Skip to contentSkip to conversation

nvmet-rdma: Suppress a class of lockdep complaints

PATCH In review Bart Van Assche 5 participants 13 messages May 9, 2023 – May 10, 2023

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nvmet-rdma: Suppress a class of lockdep complaints

Waiting in nvmet_rdma_queue_connect() for previous teardowns to complete (flush_workqueue(nvmet_wq)) confuses lockdep. Clear up this confusion by by using dynamic lockdep keys instead of static lockdep keys for queue->release_work.

See also the following report by Shinichiro:
https://lore.kernel.org/all/rsmmxrchy6voi5qhl4irss5sprna3f5owkqtvybxglcv2pnylm@xmrnpfu3tfpe/

This patch avoids that lockdep reports the following:

======================================================
WARNING: possible circular locking dependency detected
4.19.0-dbg+ #1 Not tainted
------------------------------------------------------
kworker/u12:0/7 is trying to acquire lock:
00000000c03a91d1 (&id_priv->handler_mutex){+.+.}, at: rdma_destroy_id+0x6f/0x440 [rdma_cm]

but task is already holding lock:
(work_completion)(&queue->release_work)){+.+.}, at: process_one_work+0x3c9/0x9f0 which lock already depends on the new lock.

the existing dependency chain (in reverse order) is:
-> #3 ((work_completion)(&queue->release_work)){+.+.}:

       process_one_work+0x447/0x9f0
       worker_thread+0x63/0x5a0
       kthread+0x1cf/0x1f0
       ret_from_fork+0x24/0x30
-> #2 ((wq_completion)"nvmet-rdma-delete-wq"){+.+.}:
       flush_workqueue+0xf3/0x970
       nvmet_rdma_cm_handler+0x1320/0x170f [nvmet_rdma]
       cma_ib_req_handler+0x72f/0xf90 [rdma_cm]
       cm_process_work+0x2e/0x110 [ib_cm]
       cm_work_handler+0x431e/0x50ba [ib_cm]
       process_one_work+0x481/0x9f0
       worker_thread+0x63/0x5a0
       kthread+0x1cf/0x1f0
       ret_from_fork+0x24/0x30
-> #1 (&id_priv->handler_mutex/1){+.+.}:
       __mutex_lock+0xfe/0xbe0
       mutex_lock_nested+0x1b/0x20
       cma_ib_req_handler+0x6aa/0xf90 [rdma_cm]
       cm_process_work+0x2e/0x110 [ib_cm]
       cm_work_handler+0x431e/0x50ba [ib_cm]
       process_one_work+0x481/0x9f0
       worker_thread+0x63/0x5a0
       kthread+0x1cf/0x1f0
       ret_from_fork+0x24/0x30
-> #0 (&id_priv->handler_mutex){+.+.}:
       lock_acquire+0xc5/0x200
       __mutex_lock+0xfe/0xbe0
       mutex_lock_nested+0x1b/0x20
       rdma_destroy_id+0x6f/0x440 [rdma_cm]
       nvmet_rdma_release_queue_work+0x8e/0x1b0 [nvmet_rdma]
       process_one_work+0x481/0x9f0
       worker_thread+0x63/0x5a0
       kthread+0x1cf/0x1f0
       ret_from_fork+0x24/0x30
other info that might help us debug this:
Chain exists of:
  &id_priv->handler_mutex --> (wq_completion)"nvmet-rdma-delete-wq" --> (work_completion)(&queue->release_work)

Possible unsafe locking scenario:
       CPU0                    CPU1
       ----                    ----
  lock((work_completion)(&queue->release_work));
                               lock((wq_completion)"nvmet-rdma-delete-wq");
                               lock((work_completion)(&queue->release_work));

lock(&id_priv->handler_mutex);

*** DEADLOCK ***

2 locks held by kworker/u12:0/7:
#0: 00000000272134f2 ((wq_completion)"nvmet-rdma-delete-wq"){+.+.}, at: process_one_work+0x3c9/0x9f0 #1: 0000000090531fcd ((work_completion)(&queue->release_work)){+.+.}, at: process_one_work+0x3c9/0x9f0

               stack backtrace:
CPU: 1 PID: 7 Comm: kworker/u12:0 Not tainted 4.19.0-dbg+ #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014
Workqueue: nvmet-rdma-delete-wq nvmet_rdma_release_queue_work [nvmet_rdma]
Call Trace:
 dump_stack+0x86/0xc5
 print_circular_bug.isra.32+0x20a/0x218
 __lock_acquire+0x1c68/0x1cf0
 lock_acquire+0xc5/0x200
 __mutex_lock+0xfe/0xbe0
 mutex_lock_nested+0x1b/0x20
 rdma_destroy_id+0x6f/0x440 [rdma_cm]
 nvmet_rdma_release_queue_work+0x8e/0x1b0 [nvmet_rdma]
 process_one_work+0x481/0x9f0
 worker_thread+0x63/0x5a0
 kthread+0x1cf/0x1f0
 ret_from_fork+0x24/0x30
Cc Sagi Grimberg <[email protected]>
Cc Max Gurtovoy <[email protected]>
Cc Hannes Reinecke <[email protected]>
Cc Shin'ichiro Kawasaki <[email protected]>
---
 drivers/nvme/target/rdma.c | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)
drivers/nvme/target/rdma.c +19 -1
index 4597bca43a6d..2c85a15cbe8d 100644
@@ -102,6 +102,9 @@ struct nvmet_rdma_queue {
 	struct nvmet_rdma_cmd	*cmds;
 
 	struct work_struct	release_work;
+#ifdef CONFIG_LOCKDEP
+	struct lock_class_key	key;
+#endif
 	struct list_head	rsp_wait_list;
 	struct list_head	rsp_wr_wait_list;
 	spinlock_t		rsp_wr_wait_lock;
@@ -1347,6 +1350,10 @@ static void nvmet_rdma_free_queue(struct nvmet_rdma_queue *queue)
 {
 	pr_debug("freeing queue %d\n", queue->idx);
 
+#ifdef CONFIG_LOCKDEP
+	lockdep_unregister_key(&queue->key);
+#endif
+
 	nvmet_sq_destroy(&queue->nvme_sq);
 
 	nvmet_rdma_destroy_queue_ib(queue);
@@ -1446,6 +1453,11 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev,
 	 * inside a CM callback would trigger a deadlock. (great API design..)
 	 */
 	INIT_WORK(&queue->release_work, nvmet_rdma_release_queue_work);
+#ifdef CONFIG_LOCKDEP
+	lockdep_register_key(&queue->key);
+	lockdep_init_map(&queue->release_work.lockdep_map,
+			 "nvmet_rdma_release_work", &queue->key, 0);
+#endif
 	queue->dev = ndev;
 	queue->cm_id = cm_id;
 	queue->port = port->nport;
@@ -1462,7 +1474,7 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev,
 	queue->idx = ida_alloc(&nvmet_rdma_queue_ida, GFP_KERNEL);
 	if (queue->idx < 0) {
 		ret = NVME_RDMA_CM_NO_RSC;
-		goto out_destroy_sq;
+		goto out_unreg_key;
 	}
 
 	/*
@@ -1511,6 +1523,12 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev,
 	nvmet_rdma_free_rsps(queue);
 out_ida_remove:
 	ida_free(&nvmet_rdma_queue_ida, queue->idx);
+out_unreg_key:
+#ifdef CONFIG_LOCKDEP
+	lockdep_unregister_key(&queue->key);
+#else
+	;
+#endif
 out_destroy_sq:
 	nvmet_sq_destroy(&queue->nvme_sq);
 out_free_queue:
#1 lore raw
Quoted 17 lines · Bart Van Assche

Waiting in nvmet_rdma_queue_connect() for previous teardowns to complete (flush_workqueue(nvmet_wq)) confuses lockdep. Clear up this confusion by by using dynamic lockdep keys instead of static lockdep keys for queue->release_work.

See also the following report by Shinichiro:
https://lore.kernel.org/all/rsmmxrchy6voi5qhl4irss5sprna3f5owkqtvybxglcv2pnylm@xmrnpfu3tfpe/

This patch avoids that lockdep reports the following:

======================================================
WARNING: possible circular locking dependency detected
4.19.0-dbg+ #1 Not tainted
------------------------------------------------------
kworker/u12:0/7 is trying to acquire lock:
00000000c03a91d1 (&id_priv->handler_mutex){+.+.}, at: rdma_destroy_id+0x6f/0x440 [rdma_cm]

Quoted 3 lines

but task is already holding lock:
(work_completion)(&queue->release_work)){+.+.}, at: process_one_work+0x3c9/0x9f0 which lock already depends on the new lock.

Bart, thank you very much for this immediate action after the discussion at LSF. This is encouraging. I applied the patch on top of v6.4-rc1 and ran the test case with various transports. Unfortunately, I observed kernel panics with rdma and siw transports [1][2]. Also I observed another lockdep WARN with tcp transport [3]. It looks that your fix unveiled more hidden issue/s.

[1] rdma:

[  152.976378][  T972] rdma_rxe: loaded
[  153.026858][  T952] run blktests nvme/003 at 2023-05-10 03:45:55
[  153.113227][  T989] (null): rxe_set_mtu: Set mtu to 1024
[  153.127322][  T990] PCLMULQDQ-NI instructions are not detected.
[  153.285421][  T989] infiniband ens3_rxe: set active
[  153.286007][  T989] infiniband ens3_rxe: added ens3
[  153.406495][ T1005] nvmet: adding nsid 1 to subsystem blktests-subsystem-1
[  153.416153][ T1006] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[  153.458713][  T110] ------------[ cut here ]------------
[  153.459534][  T110] WARNING: CPU: 0 PID: 110 at kernel/softirq.c:376 __local_bh_enable_ip+0xd4/0x140
[  153.460296][  T110] Modules linked in: crc32_generic rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel nvg
[  153.464985][  T110] CPU: 0 PID: 110 Comm: kworker/u8:5 Not tainted 6.4.0-rc1+ #4
[  153.465584][  T110] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/4
[  153.466370][  T110] Workqueue: rdma_cm cma_work_handler [rdma_cm]
[  153.466884][  T110] RIP: 0010:__local_bh_enable_ip+0xd4/0x140
[  153.467361][  T110] Code: 09 00 e8 5f 7b 3e 00 fb 0f 1f 44 00 00 65 8b 05 6a a9 df 5d 85 c0 74 5c 5bf
[  153.468906][  T110] RSP: 0018:ffff88811d0e7458 EFLAGS: 00010046
[  153.469398][  T110] RAX: 0000000000000000 RBX: 0000000000000201 RCX: 1ffffffff4cac45e
[  153.470022][  T110] RDX: 0000000000000000 RSI: 0000000000000201 RDI: ffffffffc1c78c5a
[  153.470649][  T110] RBP: ffffffffc1c78c5a R08: 0000000000000000 R09: ffff8881302b2afb
[  153.471313][  T110] R10: ffffed102605655f R11: ffff88811d8a3300 R12: ffff8881302b49e8
[  153.471937][  T110] R13: ffff88812eb96998 R14: ffff8881302b2000 R15: ffff88811d0e7590
[  153.472563][  T110] FS:  0000000000000000(0000) GS:ffff8883aee00000(0000) knlGS:0000000000000000
[  153.473296][  T110] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  153.473820][  T110] CR2: 0000561acc466184 CR3: 000000010b3ce000 CR4: 00000000000006f0
[  153.474448][  T110] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  153.475097][  T110] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  153.475721][  T110] Call Trace:
[  153.475986][  T110]  <TASK>
[  153.476248][  T110]  rxe_post_send+0xaa/0x29c0 [rdma_rxe]
[  153.476692][  T110]  ? do_raw_spin_lock+0x12e/0x270
[  153.477124][  T110]  ? __pfx_do_raw_spin_lock+0x10/0x10
[  153.477533][  T110]  ib_send_mad+0x81d/0x1270 [ib_core]
[  153.478011][  T110]  ? __pfx_ib_send_mad+0x10/0x10 [ib_core]
[  153.478522][  T110]  ib_post_send_mad+0x416/0x2560 [ib_core]
[  153.479023][  T110]  ? lock_acquire+0x1a9/0x4e0
[  153.479414][  T110]  ? find_held_lock+0x2d/0x110
[  153.479810][  T110]  ? __pfx_ib_post_send_mad+0x10/0x10 [ib_core]
[  153.480341][  T110]  ? cm_form_tid+0x16f/0x210 [ib_cm]
[  153.481615][  T110]  ? __pfx_lock_release+0x10/0x10
[  153.482894][  T110]  ? do_raw_spin_lock+0x12e/0x270
[  153.484175][  T110]  ? __pfx_do_raw_spin_lock+0x10/0x10
[  153.485439][  T110]  ? do_raw_spin_unlock+0x54/0x1f0
[  153.486700][  T110]  ? do_raw_spin_unlock+0x54/0x1f0
[  153.487937][  T110]  ib_send_cm_req+0x21b0/0x4730 [ib_cm]
[  153.489261][  T110]  ? __pfx_ib_send_cm_req+0x10/0x10 [ib_cm]
[  153.490554][  T110]  ? __pfx_lock_release+0x10/0x10
[  153.491806][  T110]  ? do_raw_spin_unlock+0x54/0x1f0
[  153.493009][  T110]  ? _raw_spin_unlock+0x29/0x50
[  153.494160][  T110]  rdma_connect_locked+0x80c/0x18c0 [rdma_cm]
[  153.495406][  T110]  ? __pfx_rdma_connect_locked+0x10/0x10 [rdma_cm]
[  153.496678][  T110]  ? lockdep_lock+0xca/0x1c0
[  153.497792][  T110]  ? __pfx_lockdep_lock+0x10/0x10
[  153.498943][  T110]  ? lockdep_unlock+0xf2/0x250
[  153.500063][  T110]  ? __lock_acquire+0x242c/0x5f00
[  153.501184][  T110]  ? __pfx___lock_acquire+0x10/0x10
[  153.502284][  T110]  nvme_rdma_cm_handler+0xa5b/0x2430 [nvme_rdma]
[  153.503441][  T110]  ? lock_acquire+0x1a9/0x4e0
[  153.504451][  T110]  ? find_held_lock+0x2d/0x110
[  153.505434][  T110]  ? __pfx_nvme_rdma_cm_handler+0x10/0x10 [nvme_rdma]
[  153.506562][  T110]  ? lock_release+0x378/0x650
[  153.507530][  T110]  ? cma_comp_exch+0x83/0x110 [rdma_cm]
[  153.508571][  T110]  cma_cm_event_handler+0xb2/0x2f0 [rdma_cm]
[  153.509630][  T110]  cma_work_handler+0x106/0x1b0 [rdma_cm]
[  153.510648][  T110]  process_one_work+0x7ed/0x13a0
[  153.511580][  T110]  ? __pfx_lock_acquire+0x10/0x10
[  153.512500][  T110]  ? __pfx_process_one_work+0x10/0x10
[  153.513448][  T110]  ? __pfx_do_raw_spin_lock+0x10/0x10
[  153.514395][  T110]  worker_thread+0x100/0x12c0
[  153.515285][  T110]  ? __kthread_parkme+0xc1/0x1f0
[  153.516181][  T110]  ? __pfx_worker_thread+0x10/0x10
[  153.517092][  T110]  kthread+0x2ea/0x3c0
[  153.517911][  T110]  ? __pfx_kthread+0x10/0x10
[  153.518764][  T110]  ret_from_fork+0x2c/0x50
[  153.519606][  T110]  </TASK>
[  153.520329][  T110] irq event stamp: 894255
[  153.521144][  T110] hardirqs last  enabled at (894253): [<ffffffffa498385c>] _raw_read_unlock_irqres0
[  153.522425][  T110] hardirqs last disabled at (894254): [<ffffffffa4982e6f>] _raw_spin_lock_irqsave+0
[  153.523682][  T110] softirqs last  enabled at (894234): [<ffffffffc18aacc0>] process_one_req+0x270/0]
[  153.525008][  T110] softirqs last disabled at (894255): [<ffffffffc1c78be3>] rxe_post_send+0x33/0x29]
[  153.526308][  T110] ---[ end trace 0000000000000000 ]---
[  153.534478][   T62] nvmet: creating discovery controller 1 for subsystem nqn.2014-08.org.nvmexpress..
[  153.540392][ T1007] nvme nvme1: new ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery", addr 10.0.2.150
[  163.615001][ T1022] nvme nvme1: Removing ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery"
[  163.640384][    C2] ------------[ cut here ]------------
[  163.641373][    C2] DEBUG_LOCKS_WARN_ON(1)
[  163.641381][    C2] WARNING: CPU: 2 PID: 72 at kernel/locking/lockdep.c:232 __lock_acquire+0x2903/0x5f00
[  163.643653][    C2] Modules linked in: crc32_generic rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel nvmet_rdma nvmet nvme_rdma nvme_fabrics rdma_cm iw_cm ib_cm ib_core nft_fib_inet nft_fib_ipv4 nft_fib_ipv6g
[  163.651432][    C2] CPU: 2 PID: 72 Comm: kworker/2:2 Tainted: G        W          6.4.0-rc1+ #4
[  163.652977][    C2] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/2014
[  163.654420][    C2] Workqueue: nvmet-wq nvmet_rdma_release_queue_work [nvmet_rdma]
[  163.655846][    C2] RIP: 0010:__lock_acquire+0x2903/0x5f00
[  163.657050][    C2] Code: 08 84 d2 0f 85 d0 1f 00 00 83 3d 93 f3 18 04 00 75 b4 48 c7 c6 a0 1c ce a4 48 c7 c7 40 11 ce a4 4c 89 5c 24 10 e8 ad d2 e5 ff <0f> 0b 31 db 4c 8b 5c 24 10 e9 ba fb ff ff 4c 89 1cc
[  163.660132][    C2] RSP: 0018:ffff8883aef09ce0 EFLAGS: 00010082
[  163.661476][    C2] RAX: 0000000000000000 RBX: 00000000000006cd RCX: 1ffff11075de136f
[  163.662915][    C2] RDX: 0000000000010003 RSI: 0000000000000004 RDI: 0000000000000001
[  163.664311][    C2] RBP: ffff88811b7cc218 R08: 0000000000000001 R09: ffff8883aef3084b
[  163.665758][    C2] R10: ffffed1075de6109 R11: 0000000000000001 R12: 0000000000000000
[  163.667166][    C2] R13: ffff88811b7cc1f0 R14: 00000000ffffffff R15: ffffffffa65622ec
[  163.668650][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[  163.670124][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  163.671561][    C2] CR2: 0000561acdb9d6d0 CR3: 000000010b3ce000 CR4: 00000000000006e0
[  163.673002][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  163.674440][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  163.675918][    C2] Call Trace:
[  163.677033][    C2]  <IRQ>
[  163.678113][    C2]  ? lock_acquire+0x1a9/0x4e0
[  163.679299][    C2]  ? find_held_lock+0x2d/0x110
[  163.680508][    C2]  ? __pfx___lock_acquire+0x10/0x10
[  163.681801][    C2]  ? lock_release+0x378/0x650
[  163.682998][    C2]  ? try_to_wake_up+0x102/0x1a30
[  163.684226][    C2]  ? __pfx_lock_release+0x10/0x10
[  163.685468][    C2]  lock_acquire+0x1a9/0x4e0
[  163.686688][    C2]  ? hrtimer_interrupt+0x100/0x810
[  163.687946][    C2]  ? __pfx_lock_acquire+0x10/0x10
[  163.689189][    C2]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[  163.690506][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[  163.691805][    C2]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[  163.693078][    C2]  ? __pfx_try_to_wake_up+0x10/0x10
[  163.694301][    C2]  _raw_spin_lock_irqsave+0x47/0x70
[  163.695510][    C2]  ? hrtimer_interrupt+0x100/0x810
[  163.696710][    C2]  hrtimer_interrupt+0x100/0x810
[  163.697889][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[  163.699096][    C2]  __sysvec_apic_timer_interrupt+0x146/0x3f0
[  163.700342][    C2]  sysvec_apic_timer_interrupt+0x8a/0xb0
[  163.701531][    C2]  </IRQ>
[  163.702505][    C2]  <TASK>
[  163.703455][    C2]  asm_sysvec_apic_timer_interrupt+0x16/0x20
[  163.704621][    C2] RIP: 0010:lockdep_unregister_key+0x105/0x250
[  163.705793][    C2] Code: 7c 08 84 d2 0f 85 29 01 00 00 8b 05 95 b1 19 04 85 c0 74 02 0f 0b e8 6a e7 ff ff 41 f7 c5 00 02 00 00 74 06 fb 0f 1f 44 00 00 <5b> 5d 41 5c 41 5d 41 5e e9 fe 6b 08 00 48 b8 00 00f
[  163.708701][    C2] RSP: 0018:ffff88811b817ca8 EFLAGS: 00000206
[  163.709878][    C2] RAX: 0000000000000002 RBX: ffffffffa87c2b78 RCX: 0000000000000001
[  163.711181][    C2] RDX: 0000000000000000 RSI: ffffffffa4ce1520 RDI: ffffffffa4fabf60
[  163.712451][    C2] RBP: ffff8881319b5ae0 R08: 0000000000000000 R09: ffff8883aef4626f
[  163.713705][    C2] R10: ffffed1075de8c4d R11: ffff88811b7cb300 R12: 0000000000000000
[  163.714934][    C2] R13: 0000000000000246 R14: ffffffffa894c088 R15: ffff888100e29118
[  163.716150][    C2]  ? lockdep_unregister_key+0xf6/0x250
[  163.717190][    C2]  nvmet_rdma_free_queue+0x2e/0x390 [nvmet_rdma]
[  163.718289][    C2]  nvmet_rdma_release_queue_work+0x3e/0x90 [nvmet_rdma]
[  163.719418][    C2]  process_one_work+0x7ed/0x13a0
[  163.720387][    C2]  ? __pfx_lock_acquire+0x10/0x10
[  163.721335][    C2]  ? __pfx_process_one_work+0x10/0x10
[  163.722303][    C2]  ? __pfx_do_raw_spin_lock+0x10/0x10
[  163.723253][    C2]  ? wq_worker_sleeping+0x36/0x200
[  163.724182][    C2]  worker_thread+0x100/0x12c0
[  163.725084][    C2]  ? __kthread_parkme+0xc1/0x1f0
[  163.726001][    C2]  ? __pfx_worker_thread+0x10/0x10
[  163.726928][    C2]  kthread+0x2ea/0x3c0
[  163.727762][    C2]  ? __pfx_kthread+0x10/0x10
[  163.728635][    C2]  ret_from_fork+0x2c/0x50
[  163.729488][    C2]  </TASK>
[  163.730237][    C2] irq event stamp: 17466
[  163.731066][    C2] hardirqs last  enabled at (17465): [<ffffffffa49830e4>] _raw_spin_unlock_irq+0x24/0x50
[  163.732306][    C2] hardirqs last disabled at (17466): [<ffffffffa4947a9a>] sysvec_apic_timer_interrupt+0xa/0xb0
[  163.733586][    C2] softirqs last  enabled at (17450): [<ffffffffa2249a91>] do_softirq.part.0+0xb1/0x140
[  163.734822][    C2] softirqs last disabled at (17385): [<ffffffffa2249a91>] do_softirq.part.0+0xb1/0x140
[  163.736059][    C2] ---[ end trace 0000000000000000 ]---
[  163.737030][    C2] general protection fault, probably for non-canonical address 0xdffffc0000000008: 0000 [#1] PREEMPT SMP KASAN PTI
[  163.738476][    C2] KASAN: null-ptr-deref in range [0x0000000000000040-0x0000000000000047]
[  163.739678][    C2] CPU: 2 PID: 72 Comm: kworker/2:2 Tainted: G        W          6.4.0-rc1+ #4
[  163.740920][    C2] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/2014
[  163.742226][    C2] Workqueue: nvmet-wq nvmet_rdma_release_queue_work [nvmet_rdma]
[  163.743412][    C2] RIP: 0010:__lock_acquire+0x24e0/0x5f00
[  163.744451][    C2] Code: 0f 83 a5 03 00 00 48 8d 1c 5b 48 c1 e3 06 48 81 c3 e0 6f 7c a8 48 b8 00 00 00 00 00 fc ff df 48 8d 7b 40 48 89 fa 48 c1 ea 03 <80> 3c 02 00 0f 85 15 25 00 00 48 83 7b 40 00 0f 85f
[  163.747177][    C2] RSP: 0018:ffff8883aef09ce0 EFLAGS: 00010002
[  163.748304][    C2] RAX: dffffc0000000000 RBX: 0000000000000000 RCX: 1ffff11075de136f
[  163.749575][    C2] RDX: 0000000000000008 RSI: 0000000000000004 RDI: 0000000000000040
[  163.750844][    C2] RBP: ffff88811b7cc218 R08: 0000000000000001 R09: ffff8883aef3084b
[  163.752116][    C2] R10: ffffed1075de6109 R11: ffff88811b7cb300 R12: 0000000000000000
[  163.753383][    C2] R13: ffff88811b7cc1f0 R14: 00000000ffffffff R15: ffffffffa65622ec
[  163.754668][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[  163.756022][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  163.757214][    C2] CR2: 0000561acdb9d6d0 CR3: 000000010b3ce000 CR4: 00000000000006e0
[  163.758521][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  163.759823][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  163.761126][    C2] Call Trace:
[  163.762098][    C2]  <IRQ>
[  163.763008][    C2]  ? lock_acquire+0x1a9/0x4e0
[  163.764024][    C2]  ? find_held_lock+0x2d/0x110
[  163.765038][    C2]  ? __pfx___lock_acquire+0x10/0x10
[  163.766077][    C2]  ? lock_release+0x378/0x650
[  163.767080][    C2]  ? try_to_wake_up+0x102/0x1a30
[  163.768093][    C2]  ? __pfx_lock_release+0x10/0x10
[  163.769110][    C2]  lock_acquire+0x1a9/0x4e0
[  163.770074][    C2]  ? hrtimer_interrupt+0x100/0x810
[  163.771084][    C2]  ? __pfx_lock_acquire+0x10/0x10
[  163.772082][    C2]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[  163.773133][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[  163.774144][    C2]  ? _raw_spin_unlock_irqrestore+0x35/0x60
[  163.775190][    C2]  ? __pfx_try_to_wake_up+0x10/0x10
[  163.776202][    C2]  _raw_spin_lock_irqsave+0x47/0x70
[  163.777215][    C2]  ? hrtimer_interrupt+0x100/0x810
[  163.778223][    C2]  hrtimer_interrupt+0x100/0x810
[  163.779216][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[  163.780236][    C2]  __sysvec_apic_timer_interrupt+0x146/0x3f0
[  163.781308][    C2]  sysvec_apic_timer_interrupt+0x8a/0xb0
[  163.782358][    C2]  </IRQ>
[  163.783216][    C2]  <TASK>
[  163.784068][    C2]  asm_sysvec_apic_timer_interrupt+0x16/0x20
[  163.785139][    C2] RIP: 0010:lockdep_unregister_key+0x105/0x250
[  163.786229][    C2] Code: 7c 08 84 d2 0f 85 29 01 00 00 8b 05 95 b1 19 04 85 c0 74 02 0f 0b e8 6a e7 ff ff 41 f7 c5 00 02 00 00 74 06 fb 0f 1f 44 00 00 <5b> 5d 41 5c 41 5d 41 5e e9 fe 6b 08 00 48 b8 00 00f
[  163.788971][    C2] RSP: 0018:ffff88811b817ca8 EFLAGS: 00000206
[  163.790091][    C2] RAX: 0000000000000002 RBX: ffffffffa87c2b78 RCX: 0000000000000001
[  163.791364][    C2] RDX: 0000000000000000 RSI: ffffffffa4ce1520 RDI: ffffffffa4fabf60
[  163.792626][    C2] RBP: ffff8881319b5ae0 R08: 0000000000000000 R09: ffff8883aef4626f
[  163.793870][    C2] R10: ffffed1075de8c4d R11: ffff88811b7cb300 R12: 0000000000000000
[  163.795103][    C2] R13: 0000000000000246 R14: ffffffffa894c088 R15: ffff888100e29118
[  163.796330][    C2]  ? lockdep_unregister_key+0xf6/0x250
[  163.797374][    C2]  nvmet_rdma_free_queue+0x2e/0x390 [nvmet_rdma]
[  163.798476][    C2]  nvmet_rdma_release_queue_work+0x3e/0x90 [nvmet_rdma]
[  163.799622][    C2]  process_one_work+0x7ed/0x13a0
[  163.800602][    C2]  ? __pfx_lock_acquire+0x10/0x10
[  163.801559][    C2]  ? __pfx_process_one_work+0x10/0x10
[  163.802534][    C2]  ? __pfx_do_raw_spin_lock+0x10/0x10
[  163.803494][    C2]  ? wq_worker_sleeping+0x36/0x200
[  163.804439][    C2]  worker_thread+0x100/0x12c0
[  163.805351][    C2]  ? __kthread_parkme+0xc1/0x1f0
[  163.806271][    C2]  ? __pfx_worker_thread+0x10/0x10
[  163.807200][    C2]  kthread+0x2ea/0x3c0
[  163.808043][    C2]  ? __pfx_kthread+0x10/0x10
[  163.808919][    C2]  ret_from_fork+0x2c/0x50
[  163.809776][    C2]  </TASK>
[  163.810521][    C2] Modules linked in: crc32_generic rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel nvmet_rdma nvmet nvme_rdma nvme_fabrics rdma_cm iw_cm ib_cm ib_core nft_fib_inet nft_fib_ipv4 nft_fib_ipv6g
[  163.817788][    C2] ---[ end trace 0000000000000000 ]---
[  163.818850][    C2] RIP: 0010:__lock_acquire+0x24e0/0x5f00
[  163.819928][    C2] Code: 0f 83 a5 03 00 00 48 8d 1c 5b 48 c1 e3 06 48 81 c3 e0 6f 7c a8 48 b8 00 00 00 00 00 fc ff df 48 8d 7b 40 48 89 fa 48 c1 ea 03 <80> 3c 02 00 0f 85 15 25 00 00 48 83 7b 40 00 0f 85f
[  163.822736][    C2] RSP: 0018:ffff8883aef09ce0 EFLAGS: 00010002
[  163.823900][    C2] RAX: dffffc0000000000 RBX: 0000000000000000 RCX: 1ffff11075de136f
[  163.825209][    C2] RDX: 0000000000000008 RSI: 0000000000000004 RDI: 0000000000000040
[  163.826521][    C2] RBP: ffff88811b7cc218 R08: 0000000000000001 R09: ffff8883aef3084b
[  163.827836][    C2] R10: ffffed1075de6109 R11: ffff88811b7cb300 R12: 0000000000000000
[  163.829145][    C2] R13: ffff88811b7cc1f0 R14: 00000000ffffffff R15: ffffffffa65622ec
[  163.830463][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[  163.831867][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  163.833105][    C2] CR2: 0000561acdb9d6d0 CR3: 000000010b3ce000 CR4: 00000000000006e0
[  163.834446][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  163.835794][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  163.837137][    C2] Kernel panic - not syncing: Fatal exception in interrupt
[  163.838585][    C2] Kernel Offset: 0x21000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
[  163.840211][    C2] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]---

[2] siw:

[   29.856520][  T498] systemd-journald[498]: File /var/log/journal/02cb4e6014a94b2fa4.
[   53.344964][  T929] rdma_rxe: loaded
[   53.400627][  T909] run blktests nvme/003 at 2023-05-10 04:10:11
[   53.520087][  T942] SoftiWARP attached
[   53.621029][  T963] nvmet: adding nsid 1 to subsystem blktests-subsystem-1
[   53.631333][  T964] iwpm_register_pid: Unable to send a nlmsg (client = 2)
[   53.634192][  T964] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[   53.687380][   T62] nvmet: creating discovery controller 1 for subsystem nqn.2014-08.org.nvmexpress..
[   53.693404][  T965] nvme nvme1: new ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery", addr 10.0.2.150
[   63.762719][  T978] nvme nvme1: Removing ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery"
[   63.790054][    C2] ------------[ cut here ]------------
[   63.790565][    C2] DEBUG_LOCKS_WARN_ON(1)
[   63.790578][    C2] WARNING: CPU: 2 PID: 72 at kernel/locking/lockdep.c:232 __lock_acquire+0x2903/0x0
[   63.791589][    C2] Modules linked in: siw rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel nvmet_rdma ng
[   63.795785][    C2] CPU: 2 PID: 72 Comm: kworker/2:2 Not tainted 6.4.0-rc1+ #4
[   63.796324][    C2] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/4
[   63.797026][    C2] Workqueue: nvmet-wq nvmet_rdma_release_queue_work [nvmet_rdma]
[   63.797607][    C2] RIP: 0010:__lock_acquire+0x2903/0x5f00
[   63.798020][    C2] Code: 08 84 d2 0f 85 d0 1f 00 00 83 3d 93 f3 18 04 00 75 b4 48 c7 c6 a0 1c ce 92c
[   63.799425][    C2] RSP: 0018:ffff8883aef09ce0 EFLAGS: 00010082
[   63.799869][    C2] RAX: 0000000000000000 RBX: 00000000000006bc RCX: 1ffff11075de136f
[   63.800640][    C2] RDX: 0000000000010003 RSI: 0000000000000004 RDI: 0000000000000001
[   63.801217][    C2] RBP: ffff88811bb54218 R08: 0000000000000001 R09: ffff8883aef3084b
[   63.801853][    C2] R10: ffffed1075de6109 R11: 0000000000000001 R12: 0000000000000000
[   63.802426][    C2] R13: ffff88811bb541f0 R14: 00000000ffffffff R15: ffffffff945622ec
[   63.803003][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[   63.803645][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   63.804120][    C2] CR2: 00007ff6323cc364 CR3: 000000021ca6c000 CR4: 00000000000006e0
[   63.804697][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[   63.805269][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[   63.805847][    C2] Call Trace:
[   63.806088][    C2]  <IRQ>
[   63.806299][    C2]  ? __pfx_mark_lock+0x10/0x10
[   63.806652][    C2]  ? mark_lock+0x101/0x16d0
[   63.806982][    C2]  ? __pfx___lock_acquire+0x10/0x10
[   63.807358][    C2]  ? lock_acquire+0x1a9/0x4e0
[   63.807701][    C2]  ? find_held_lock+0x2d/0x110
[   63.808949][    C2]  lock_acquire+0x1a9/0x4e0
[   63.810274][    C2]  ? hrtimer_interrupt+0x100/0x810
[   63.811531][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   63.812784][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[   63.814043][    C2]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[   63.815341][    C2]  _raw_spin_lock_irqsave+0x47/0x70
[   63.816569][    C2]  ? hrtimer_interrupt+0x100/0x810
[   63.817761][    C2]  hrtimer_interrupt+0x100/0x810
[   63.818942][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[   63.820140][    C2]  ? __pfx___do_softirq+0x10/0x10
[   63.821316][    C2]  __sysvec_apic_timer_interrupt+0x146/0x3f0
[   63.822564][    C2]  sysvec_apic_timer_interrupt+0x8a/0xb0
[   63.823777][    C2]  </IRQ>
[   63.824793][    C2]  <TASK>
[   63.825779][    C2]  asm_sysvec_apic_timer_interrupt+0x16/0x20
[   63.826962][    C2] RIP: 0010:lockdep_unregister_key+0x105/0x250
[   63.828138][    C2] Code: 7c 08 84 d2 0f 85 29 01 00 00 8b 05 95 b1 19 04 85 c0 74 02 0f 0b e8 6a e7f
[   63.831047][    C2] RSP: 0018:ffff88811bb9fca8 EFLAGS: 00000206
[   63.832253][    C2] RAX: 0000000000000002 RBX: ffffffff967c2b78 RCX: 0000000000000001
[   63.833595][    C2] RDX: 0000000000000000 RSI: ffffffff92ce1520 RDI: ffffffff92fabf60
[   63.834913][    C2] RBP: ffff888130ea22e0 R08: 0000000000000000 R09: ffff8883aef4626f
[   63.836204][    C2] R10: ffffed1075de8c4d R11: ffff88811bb53300 R12: ffff8881010df128
[   63.837473][    C2] R13: 0000000000000246 R14: ffffffff9694e678 R15: ffff88810112f918
[   63.838724][    C2]  ? lockdep_unregister_key+0xf6/0x250
[   63.839775][    C2]  nvmet_rdma_free_queue+0x2e/0x390 [nvmet_rdma]
[   63.840879][    C2]  nvmet_rdma_release_queue_work+0x3e/0x90 [nvmet_rdma]
[   63.842021][    C2]  process_one_work+0x7ed/0x13a0
[   63.843016][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   63.844005][    C2]  ? __pfx_process_one_work+0x10/0x10
[   63.844997][    C2]  ? __pfx_do_raw_spin_lock+0x10/0x10
[   63.845956][    C2]  worker_thread+0x100/0x12c0
[   63.846864][    C2]  ? __kthread_parkme+0xc1/0x1f0
[   63.847782][    C2]  ? __pfx_worker_thread+0x10/0x10
[   63.848700][    C2]  kthread+0x2ea/0x3c0
[   63.849542][    C2]  ? __pfx_kthread+0x10/0x10
[   63.850426][    C2]  ret_from_fork+0x2c/0x50
[   63.851296][    C2]  </TASK>
[   63.852054][    C2] irq event stamp: 231644
[   63.852885][    C2] hardirqs last  enabled at (231643): [<ffffffff929830e4>] _raw_spin_unlock_irq+0x0
[   63.854124][    C2] hardirqs last disabled at (231644): [<ffffffff92947a9a>] sysvec_apic_timer_inter0
[   63.855405][    C2] softirqs last  enabled at (231330): [<ffffffff9024986e>] __irq_exit_rcu+0xfe/0x20
[   63.856630][    C2] softirqs last disabled at (231303): [<ffffffff9024986e>] __irq_exit_rcu+0xfe/0x20
[   63.857837][    C2] ---[ end trace 0000000000000000 ]---
[   63.858798][    C2] general protection fault, probably for non-canonical address 0xdffffc0000000008:I
[   63.860227][    C2] KASAN: null-ptr-deref in range [0x0000000000000040-0x0000000000000047]
[   63.861411][    C2] CPU: 2 PID: 72 Comm: kworker/2:2 Tainted: G        W          6.4.0-rc1+ #4
[   63.862645][    C2] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/4
[   63.863940][    C2] Workqueue: nvmet-wq nvmet_rdma_release_queue_work [nvmet_rdma]
[   63.865117][    C2] RIP: 0010:__lock_acquire+0x24e0/0x5f00
[   63.866150][    C2] Code: 0f 83 a5 03 00 00 48 8d 1c 5b 48 c1 e3 06 48 81 c3 e0 6f 7c 96 48 b8 00 00f
[   63.868877][    C2] RSP: 0018:ffff8883aef09ce0 EFLAGS: 00010002
[   63.869989][    C2] RAX: dffffc0000000000 RBX: 0000000000000000 RCX: 1ffff11075de136f
[   63.871252][    C2] RDX: 0000000000000008 RSI: 0000000000000004 RDI: 0000000000000040
[   63.872513][    C2] RBP: ffff88811bb54218 R08: 0000000000000001 R09: ffff8883aef3084b
[   63.873777][    C2] R10: ffffed1075de6109 R11: ffff88811bb53300 R12: 0000000000000000
[   63.875044][    C2] R13: ffff88811bb541f0 R14: 00000000ffffffff R15: ffffffff945622ec
[   63.876317][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[   63.877674][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   63.878862][    C2] CR2: 00007ff6323cc364 CR3: 000000021ca6c000 CR4: 00000000000006e0
[   63.880158][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[   63.881453][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[   63.882754][    C2] Call Trace:
[   63.883721][    C2]  <IRQ>
[   63.884655][    C2]  ? __pfx_mark_lock+0x10/0x10
[   63.885705][    C2]  ? mark_lock+0x101/0x16d0
[   63.886702][    C2]  ? __pfx___lock_acquire+0x10/0x10
[   63.887744][    C2]  ? lock_acquire+0x1a9/0x4e0
[   63.888740][    C2]  ? find_held_lock+0x2d/0x110
[   63.889742][    C2]  lock_acquire+0x1a9/0x4e0
[   63.890711][    C2]  ? hrtimer_interrupt+0x100/0x810
[   63.891728][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   63.892735][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[   63.893753][    C2]  ? _raw_spin_unlock_irqrestore+0x4c/0x60
[   63.894809][    C2]  _raw_spin_lock_irqsave+0x47/0x70
[   63.895815][    C2]  ? hrtimer_interrupt+0x100/0x810
[   63.896803][    C2]  hrtimer_interrupt+0x100/0x810
[   63.897789][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[   63.898802][    C2]  ? __pfx___do_softirq+0x10/0x10
[   63.899798][    C2]  __sysvec_apic_timer_interrupt+0x146/0x3f0
[   63.900865][    C2]  sysvec_apic_timer_interrupt+0x8a/0xb0
[   63.901908][    C2]  </IRQ>
[   63.902758][    C2]  <TASK>
[   63.903603][    C2]  asm_sysvec_apic_timer_interrupt+0x16/0x20
[   63.904674][    C2] RIP: 0010:lockdep_unregister_key+0x105/0x250
[   63.905757][    C2] Code: 7c 08 84 d2 0f 85 29 01 00 00 8b 05 95 b1 19 04 85 c0 74 02 0f 0b e8 6a e7f
[   63.908505][    C2] RSP: 0018:ffff88811bb9fca8 EFLAGS: 00000206
[   63.909632][    C2] RAX: 0000000000000002 RBX: ffffffff967c2b78 RCX: 0000000000000001
[   63.910895][    C2] RDX: 0000000000000000 RSI: ffffffff92ce1520 RDI: ffffffff92fabf60
[   63.912148][    C2] RBP: ffff888130ea22e0 R08: 0000000000000000 R09: ffff8883aef4626f
[   63.913415][    C2] R10: ffffed1075de8c4d R11: ffff88811bb53300 R12: ffff8881010df128
[   63.914677][    C2] R13: 0000000000000246 R14: ffffffff9694e678 R15: ffff88810112f918
[   63.915926][    C2]  ? lockdep_unregister_key+0xf6/0x250
[   63.916985][    C2]  nvmet_rdma_free_queue+0x2e/0x390 [nvmet_rdma]
[   63.918102][    C2]  nvmet_rdma_release_queue_work+0x3e/0x90 [nvmet_rdma]
[   63.919251][    C2]  process_one_work+0x7ed/0x13a0
[   63.920249][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   63.921242][    C2]  ? __pfx_process_one_work+0x10/0x10
[   63.922239][    C2]  ? __pfx_do_raw_spin_lock+0x10/0x10
[   63.923203][    C2]  worker_thread+0x100/0x12c0
[   63.924115][    C2]  ? __kthread_parkme+0xc1/0x1f0
[   63.925038][    C2]  ? __pfx_worker_thread+0x10/0x10
[   63.925970][    C2]  kthread+0x2ea/0x3c0
[   63.926825][    C2]  ? __pfx_kthread+0x10/0x10
[   63.927713][    C2]  ret_from_fork+0x2c/0x50
[   63.928584][    C2]  </TASK>
[   63.929337][    C2] Modules linked in: siw rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel nvmet_rdma ng
[   63.936492][    C2] ---[ end trace 0000000000000000 ]---
[   63.937541][    C2] RIP: 0010:__lock_acquire+0x24e0/0x5f00
[   63.938610][    C2] Code: 0f 83 a5 03 00 00 48 8d 1c 5b 48 c1 e3 06 48 81 c3 e0 6f 7c 96 48 b8 00 00f
[   63.941394][    C2] RSP: 0018:ffff8883aef09ce0 EFLAGS: 00010002
[   63.942548][    C2] RAX: dffffc0000000000 RBX: 0000000000000000 RCX: 1ffff11075de136f
[   63.943849][    C2] RDX: 0000000000000008 RSI: 0000000000000004 RDI: 0000000000000040
[   63.945140][    C2] RBP: ffff88811bb54218 R08: 0000000000000001 R09: ffff8883aef3084b
[   63.946442][    C2] R10: ffffed1075de6109 R11: ffff88811bb53300 R12: 0000000000000000
[   63.947746][    C2] R13: ffff88811bb541f0 R14: 00000000ffffffff R15: ffffffff945622ec
[   63.949056][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[   63.950441][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   63.951673][    C2] CR2: 00007ff6323cc364 CR3: 000000021ca6c000 CR4: 00000000000006e0
[   63.953009][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[   63.954336][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[   63.955667][    C2] Kernel panic - not syncing: Fatal exception in interrupt
[   63.957088][    C2] Kernel Offset: 0xf000000 from 0xffffffff81000000 (relocation range: 0xffffffff80)
[   63.958706][    C2] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]---

[3] tcp:

[   56.043999][  T910] run blktests nvme/003 at 2023-05-10 05:27:59
[   56.138547][  T943] nvmet: adding nsid 1 to subsystem blktests-subsystem-1
[   56.148463][  T944] nvmet_tcp: enabling port 0 (127.0.0.1:4420)
[   56.195051][  T170] nvmet: creating discovery controller 1 for subsystem nqn.2014-08.org.nvmexpress..
[   56.200155][  T945] nvme nvme1: new ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery", addr 127.0.0.10
[   66.282334][  T956] nvme nvme1: Removing ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery"
[   66.286070][  T169]
[   66.286362][  T169] ======================================================
[   66.287082][  T169] WARNING: possible circular locking dependency detected
[   66.287903][  T169] 6.4.0-rc1+ #4 Not tainted
[   66.288447][  T169] ------------------------------------------------------
[   66.289266][  T169] kworker/0:3/169 is trying to acquire lock:
[   66.289938][  T169] ffff888108c09030 ((work_completion)(&queue->io_work)){+.+.}-{0:0}, at: __flush_w0
[   66.291264][  T169]
[   66.291264][  T169] but task is already holding lock:
[   66.292058][  T169] ffff88811e6efdb8 ((work_completion)(&queue->release_work)){+.+.}-{0:0}, at: proc0
[   66.293361][  T169]
[   66.293361][  T169] which lock already depends on the new lock.
[   66.293361][  T169]
[   66.294496][  T169]
[   66.294496][  T169] the existing dependency chain (in reverse order) is:
[   66.295497][  T169]
[   66.295497][  T169] -> #2 ((work_completion)(&queue->release_work)){+.+.}-{0:0}:
[   66.296581][  T169]        process_one_work+0x786/0x13a0
[   66.297205][  T169]        worker_thread+0x100/0x12c0
[   66.297766][  T169]        kthread+0x2ea/0x3c0
[   66.298287][  T169]        ret_from_fork+0x2c/0x50
[   66.298843][  T169]
[   66.298843][  T169] -> #1 ((wq_completion)nvmet-wq){+.+.}-{0:0}:
[   66.299765][  T169]        __flush_workqueue+0x130/0x12d0
[   66.300396][  T169]        nvmet_tcp_install_queue+0x6e/0x2a0 [nvmet_tcp]
[   66.301182][  T169]        nvmet_install_queue+0x331/0x830 [nvmet]
[   66.301920][  T169]        nvmet_execute_admin_connect+0x34e/0x7f0 [nvmet]
[   66.302735][  T169]        nvmet_tcp_io_work+0x26a7/0x3ae0 [nvmet_tcp]
[   66.303494][  T169]        process_one_work+0x7ed/0x13a0
[   66.304129][  T169]        worker_thread+0x100/0x12c0
[   66.304734][  T169]        kthread+0x2ea/0x3c0
[   66.305286][  T169]        ret_from_fork+0x2c/0x50
[   66.305833][  T169]
[   66.305833][  T169] -> #0 ((work_completion)(&queue->io_work)){+.+.}-{0:0}:
[   66.306882][  T169]        __lock_acquire+0x2f25/0x5f00
[   66.307528][  T169]        lock_acquire+0x1a9/0x4e0
[   66.308092][  T169]        __flush_work+0xf7/0x8f0
[   66.309409][  T169]        __cancel_work_timer+0x2ad/0x3e0
[   66.310715][  T169]        nvmet_tcp_release_queue_work+0x2e3/0x1330 [nvmet_tcp]
[   66.312220][  T169]        process_one_work+0x7ed/0x13a0
[   66.313537][  T169]        worker_thread+0x100/0x12c0
[   66.314784][  T169]        kthread+0x2ea/0x3c0
[   66.315974][  T169]        ret_from_fork+0x2c/0x50
[   66.317214][  T169]
[   66.317214][  T169] other info that might help us debug this:
[   66.317214][  T169]
[   66.320432][  T169] Chain exists of:
[   66.320432][  T169]   (work_completion)(&queue->io_work) --> (wq_completion)nvmet-wq --> (work_compl)
[   66.320432][  T169]
[   66.324324][  T169]  Possible unsafe locking scenario:
[   66.324324][  T169]
[   66.326482][  T169]        CPU0                    CPU1
[   66.327690][  T169]        ----                    ----
[   66.328907][  T169]   lock((work_completion)(&queue->release_work));
[   66.330282][  T169]                                lock((wq_completion)nvmet-wq);
[   66.331700][  T169]                                lock((work_completion)(&queue->release_work));
[   66.333328][  T169]   lock((work_completion)(&queue->io_work));
[   66.334587][  T169]
[   66.334587][  T169]  *** DEADLOCK ***
[   66.334587][  T169]
[   66.337089][  T169] 2 locks held by kworker/0:3/169:
[   66.338208][  T169]  #0: ffff88812557f938 ((wq_completion)nvmet-wq){+.+.}-{0:0}, at: process_one_wor0
[   66.339961][  T169]  #1: ffff88811e6efdb8 ((work_completion)(&queue->release_work)){+.+.}-{0:0}, at:0
[   66.341880][  T169]
[   66.341880][  T169] stack backtrace:
[   66.343686][  T169] CPU: 0 PID: 169 Comm: kworker/0:3 Not tainted 6.4.0-rc1+ #4
[   66.345105][  T169] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/4
[   66.346774][  T169] Workqueue: nvmet-wq nvmet_tcp_release_queue_work [nvmet_tcp]
[   66.348254][  T169] Call Trace:
[   66.349274][  T169]  <TASK>
[   66.350217][  T169]  dump_stack_lvl+0x57/0x90
[   66.351357][  T169]  check_noncircular+0x27b/0x310
[   66.352522][  T169]  ? __pfx_mark_lock+0x10/0x10
[   66.353633][  T169]  ? __pfx_check_noncircular+0x10/0x10
[   66.354841][  T169]  ? register_lock_class+0xba/0x10a0
[   66.356031][  T169]  ? ret_from_fork+0x2c/0x50
[   66.357207][  T169]  ? lockdep_lock+0xca/0x1c0
[   66.358328][  T169]  ? __pfx_lockdep_lock+0x10/0x10
[   66.359480][  T169]  __lock_acquire+0x2f25/0x5f00
[   66.360621][  T169]  ? __pfx___lock_acquire+0x10/0x10
[   66.361784][  T169]  ? save_trace+0x53/0xbe0
[   66.362902][  T169]  lock_acquire+0x1a9/0x4e0
[   66.364034][  T169]  ? __flush_work+0xd7/0x8f0
[   66.365152][  T169]  ? __pfx_lock_acquire+0x10/0x10
[   66.366420][  T169]  ? __pfx___lock_acquire+0x10/0x10
[   66.367563][  T169]  __flush_work+0xf7/0x8f0
[   66.368641][  T169]  ? __flush_work+0xd7/0x8f0
[   66.369734][  T169]  ? __pfx___flush_work+0x10/0x10
[   66.370884][  T169]  ? find_held_lock+0x2d/0x110
[   66.372006][  T169]  ? mark_held_locks+0x96/0xe0
[   66.373121][  T169]  __cancel_work_timer+0x2ad/0x3e0
[   66.374307][  T169]  ? __pfx___cancel_work_timer+0x10/0x10
[   66.375729][  T169]  ? nvmet_tcp_release_queue_work+0x2cf/0x1330 [nvmet_tcp]
[   66.377106][  T169]  ? mark_held_locks+0x96/0xe0
[   66.378233][  T169]  ? __pfx_do_raw_write_lock+0x10/0x10
[   66.379384][  T169]  ? __local_bh_enable_ip+0xb1/0x140
[   66.380646][  T169]  ? lockdep_hardirqs_on+0x7d/0x100
[   66.381842][  T169]  ? __pfx_sk_stream_write_space+0x10/0x10
[   66.383016][  T169]  nvmet_tcp_release_queue_work+0x2e3/0x1330 [nvmet_tcp]
[   66.384342][  T169]  process_one_work+0x7ed/0x13a0
[   66.385395][  T169]  ? __pfx_lock_acquire+0x10/0x10
[   66.386463][  T169]  ? __pfx_process_one_work+0x10/0x10
[   66.387597][  T169]  ? __pfx_do_raw_spin_lock+0x10/0x10
[   66.388733][  T169]  worker_thread+0x100/0x12c0
[   66.389796][  T169]  ? __kthread_parkme+0xc1/0x1f0
[   66.390866][  T169]  ? __pfx_worker_thread+0x10/0x10
[   66.391958][  T169]  kthread+0x2ea/0x3c0
[   66.392933][  T169]  ? __pfx_kthread+0x10/0x10
[   66.393974][  T169]  ret_from_fork+0x2c/0x50
[   66.394995][  T169]  </TASK>
#2 lore raw
Quoted 5 lines · Shinichiro Kawasaki

Bart, thank you very much for this immediate action after the discussion at LSF. This is encouraging. I applied the patch on top of v6.4-rc1 and ran the test case with various transports. Unfortunately, I observed kernel panics with rdma and siw transports [1][2]. Also I observed another lockdep WARN with tcp transport [3]. It looks that your fix unveiled more hidden issue/s.

Please use siw instead of rxe when running blktests - there are known issues with the rxe driver.

Please apply these patches on top of kernel v6.3 instead of v6.4-rc1. The hrtimer_interrupt() crash shown below is a v6.4-rc1 regression and does not occur with the v6.3 kernel.

Since my patch is for the RDMA transport, it is not clear to me why a report for the TCP transport is included in a reply to my patch?

Thanks,

Bart.

maintainer
#3 lore raw
Quoted 7 lines
Quoted 6 lines

Bart, thank you very much for this immediate action after the discussion at LSF. This is encouraging. I applied the patch on top of v6.4-rc1 and ran the test case with various transports. Unfortunately, I observed kernel panics with rdma and siw transports [1][2]. Also I observed another lockdep WARN with tcp transport [3]. It looks that your fix unveiled more hidden issue/s.

Quoted 5 lines

Please use siw instead of rxe when running blktests - there are known issues with the rxe driver.

Please apply these patches on top of kernel v6.3 instead of v6.4-rc1. The hrtimer_interrupt() crash shown below is a v6.4-rc1 regression and does not occur with the v6.3 kernel.

Since my patch is for the RDMA transport, it is not clear to me why a report for the TCP transport is included in a reply to my patch?

Agree,

The same fundamental issue exists in nvmet-tcp as well.

Copying to nvmet-tcp (compile-tested only) gives:
--

drivers/nvme/target/tcp.c +12 -1
index ed98df72c76b..a4367fa6f55b 100644
@@ -163,7 +163,9 @@ struct nvmet_tcp_queue {
        struct sockaddr_storage sockaddr;
        struct sockaddr_storage sockaddr_peer;
        struct work_struct      release_work;
-
+#ifdef CONFIG_LOCKDEP
+       struct lock_class_key   key;
+#endif
        int                     idx;
        struct list_head        queue_list;
 
@@ -1472,6 +1474,10 @@ static void nvmet_tcp_release_queue_work(struct work_struct *w)
        list_del_init(&queue->queue_list);
        mutex_unlock(&nvmet_tcp_queue_mutex);
 
+#ifdef CONFIG_LOCKDEP
+       lockdep_unregister_key(&queue->key);
+#endif
+
        nvmet_tcp_restore_socket_callbacks(queue);
        cancel_work_sync(&queue->io_work);
        /* stop accepting incoming data */
@@ -1654,6 +1660,11 @@ static int nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
        if (ret)
                goto out_destroy_sq;
 
+#ifdef CONFIG_LOCKDEP
+       lockdep_register_key(&queue->key);
+       lockdep_init_map(&queue->release_work.lockdep_map,
+                        "nvmet_tcp_release_work", &queue->key, 0);
+#endif
        return 0;
 out_destroy_sq:
        mutex_lock(&nvmet_tcp_queue_mutex);

--

#4 lore raw
Quoted 13 lines · Sagi Grimberg
Quoted 13 lines
Quoted 9 lines

Bart, thank you very much for this immediate action after the discussion at LSF.
This is encouraging. I applied the patch on top of v6.4-rc1 and ran the test
case with various transports. Unfortunately, I observed kernel panics with rdma
and siw transports [1][2]. Also I observed another lockdep WARN with tcp transport [3]. It looks that your fix unveiled more hidden issue/s.

Please use siw instead of rxe when running blktests - there are known issues with the rxe driver.

Quoted 8 lines
Quoted 6 lines

Please apply these patches on top of kernel v6.3 instead of v6.4-rc1. The hrtimer_interrupt() crash shown below is a v6.4-rc1 regression and does not occur with the v6.3 kernel.

Since my patch is for the RDMA transport, it is not clear to me why a report for the TCP transport is included in a reply to my patch?

Agree,

Sorry for my misunderstandings. I've tested again with siw and kernel v6.3. Still I see the kernel panic. Here's the kernel messages I've got.

[   59.567730][  T935] rdma_rxe: loaded
[   59.614648][  T915] run blktests nvme/003 at 2023-05-10 08:48:26
[   59.714402][  T948] SoftiWARP attached
[   59.801368][  T969] nvmet: adding nsid 1 to subsystem blktests-subsystem-1
[   59.810654][  T970] iwpm_register_pid: Unable to send a nlmsg (client = 2)
[   59.813025][  T970] nvmet_rdma: enabling port 0 (10.0.2.15:4420)
[   59.861254][   T61] nvmet: creating discovery controller 1 for subsystem nqn.2014-08.org.nvmexpress..
[   59.869998][  T971] nvme nvme1: new ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery", addr 10.0.2.150
[   69.939259][  T982] nvme nvme1: Removing ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery"
[   69.963920][    C2] ------------[ cut here ]------------
[   69.964368][    C2] DEBUG_LOCKS_WARN_ON(1)
[   69.964382][    C2] WARNING: CPU: 2 PID: 825 at kernel/locking/lockdep.c:232 __lock_acquire+0x28a4/00
[   69.965436][    C2] Modules linked in: siw rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel nvmet_rdma ng
[   69.969849][    C2] CPU: 2 PID: 825 Comm: kworker/2:4 Not tainted 6.3.0+ #5
[   69.970389][    C2] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/4
[   69.971100][    C2] Workqueue: nvmet-wq nvmet_rdma_release_queue_work [nvmet_rdma]
[   69.971671][    C2] RIP: 0010:__lock_acquire+0x28a4/0x5eb0
[   69.972090][    C2] Code: 08 84 d2 0f 85 52 22 00 00 83 3d 42 c2 18 04 00 75 b4 48 c7 c6 20 17 ce 85b
[   69.973525][    C2] RSP: 0018:ffff8883aef09ce8 EFLAGS: 00010092
[   69.973980][    C2] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 1ffff11075de1370
[   69.974565][    C2] RDX: 0000000000010003 RSI: 0000000000000004 RDI: 0000000000000001
[   69.975155][    C2] RBP: ffff8881215bc108 R08: 0000000000000001 R09: ffff8883aef3084b
[   69.975744][    C2] R10: ffffed1075de6109 R11: 0000000000000001 R12: 0000000000000002
[   69.976331][    C2] R13: 0000000000000000 R14: ffffffff8755632c R15: 00000000ffffffff
[   69.976922][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[   69.977576][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   69.978063][    C2] CR2: 0000563bf892e000 CR3: 000000012eb74000 CR4: 00000000000006e0
[   69.978651][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[   69.979241][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[   69.979830][    C2] Call Trace:
[   69.980074][    C2]  <IRQ>
[   69.980290][    C2]  ? lock_acquire+0x1b7/0x4e0
[   69.980641][    C2]  ? __pfx___lock_acquire+0x10/0x10
[   69.981030][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   69.981402][    C2]  ? update_process_times+0x158/0x1d0
[   69.981804][    C2]  ? __pfx___lock_acquire+0x10/0x10
[   69.982192][    C2]  lock_acquire+0x1a7/0x4e0
[   69.982528][    C2]  ? hrtimer_interrupt+0x100/0x810
[   69.983794][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   69.985043][    C2]  ? hrtimer_interrupt+0x339/0x810
[   69.986262][    C2]  ? kvm_clock_read+0x14/0x30
[   69.987449][    C2]  _raw_spin_lock_irqsave+0x47/0x70
[   69.988673][    C2]  ? hrtimer_interrupt+0x100/0x810
[   69.989922][    C2]  hrtimer_interrupt+0x100/0x810
[   69.991111][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[   69.992319][    C2]  __sysvec_apic_timer_interrupt+0x146/0x3f0
[   69.993581][    C2]  sysvec_apic_timer_interrupt+0x8a/0xb0
[   69.994853][    C2]  </IRQ>
[   69.995882][    C2]  <TASK>
[   69.996900][    C2]  asm_sysvec_apic_timer_interrupt+0x16/0x20
[   69.998122][    C2] RIP: 0010:lockdep_unregister_key+0x105/0x250
[   69.999323][    C2] Code: 7c 08 84 d2 0f 85 29 01 00 00 8b 05 75 7e 19 04 85 c0 74 02 0f 0b e8 6a e7f
[   70.002306][    C2] RSP: 0018:ffff88810ad67ca0 EFLAGS: 00000206
[   70.003521][    C2] RAX: 0000000000000002 RBX: ffffffff897afb78 RCX: 0000000000000001
[   70.004921][    C2] RDX: 0000000000000000 RSI: ffffffff85ce0fa0 RDI: ffffffff85fa7100
[   70.006259][    C2] RBP: ffff88812da24ae0 R08: 0000000000000000 R09: ffff8883aef45c2f
[   70.007590][    C2] R10: ffffed1075de8b85 R11: ffff8881215bb280 R12: 0000000000000000
[   70.008924][    C2] R13: 0000000000000246 R14: ffffffff89939550 R15: ffff88812f5c3118
[   70.010209][    C2]  nvmet_rdma_free_queue+0x2e/0x390 [nvmet_rdma]
[   70.011363][    C2]  nvmet_rdma_release_queue_work+0x3e/0x90 [nvmet_rdma]
[   70.012535][    C2]  process_one_work+0x7e4/0x1390
[   70.013568][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   70.014594][    C2]  ? __pfx_process_one_work+0x10/0x10
[   70.015661][    C2]  ? __pfx_do_raw_spin_lock+0x10/0x10
[   70.016745][    C2]  worker_thread+0xf7/0x12b0
[   70.017730][    C2]  ? __kthread_parkme+0xc1/0x1f0
[   70.018687][    C2]  ? __pfx_worker_thread+0x10/0x10
[   70.019645][    C2]  kthread+0x29e/0x340
[   70.020517][    C2]  ? __pfx_kthread+0x10/0x10
[   70.021427][    C2]  ret_from_fork+0x2c/0x50
[   70.022323][    C2]  </TASK>
[   70.023111][    C2] irq event stamp: 3790
[   70.023976][    C2] hardirqs last  enabled at (3789): [<ffffffff85a00e86>] asm_sysvec_apic_timer_int0
[   70.025311][    C2] hardirqs last disabled at (3790): [<ffffffff85907a9a>] sysvec_apic_timer_interru0
[   70.026609][    C2] softirqs last  enabled at (3158): [<ffffffff832483be>] __irq_exit_rcu+0xfe/0x260
[   70.027887][    C2] softirqs last disabled at (3151): [<ffffffff832483be>] __irq_exit_rcu+0xfe/0x260
[   70.029108][    C2] ---[ end trace 0000000000000000 ]---
[   70.030101][    C2] general protection fault, probably for non-canonical address 0xdffffc0000000008:I
[   70.031556][    C2] KASAN: null-ptr-deref in range [0x0000000000000040-0x0000000000000047]
[   70.032794][    C2] CPU: 2 PID: 825 Comm: kworker/2:4 Tainted: G        W          6.3.0+ #5
[   70.034045][    C2] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.2-1.fc38 04/01/4
[   70.035360][    C2] Workqueue: nvmet-wq nvmet_rdma_release_queue_work [nvmet_rdma]
[   70.036559][    C2] RIP: 0010:__lock_acquire+0x2481/0x5eb0
[   70.037617][    C2] Code: 0f 83 a7 03 00 00 48 8d 1c 5b 48 c1 e3 06 48 81 c3 e0 3f 7b 89 48 b8 00 00f
[   70.040455][    C2] RSP: 0018:ffff8883aef09ce8 EFLAGS: 00010002
[   70.041629][    C2] RAX: dffffc0000000000 RBX: 0000000000000000 RCX: 1ffff11075de1370
[   70.042955][    C2] RDX: 0000000000000008 RSI: 0000000000000004 RDI: 0000000000000040
[   70.044233][    C2] RBP: ffff8881215bc108 R08: 0000000000000001 R09: ffff8883aef3084b
[   70.045519][    C2] R10: ffffed1075de6109 R11: ffff8881215bb280 R12: 0000000000000002
[   70.046855][    C2] R13: 0000000000000000 R14: ffffffff8755632c R15: 00000000ffffffff
[   70.048148][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[   70.049519][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   70.050776][    C2] CR2: 0000563bf892e000 CR3: 000000012eb74000 CR4: 00000000000006e0
[   70.052120][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[   70.053428][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[   70.054774][    C2] Call Trace:
[   70.055795][    C2]  <IRQ>
[   70.056779][    C2]  ? lock_acquire+0x1b7/0x4e0
[   70.057890][    C2]  ? __pfx___lock_acquire+0x10/0x10
[   70.058980][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   70.060031][    C2]  ? update_process_times+0x158/0x1d0
[   70.061089][    C2]  ? __pfx___lock_acquire+0x10/0x10
[   70.062128][    C2]  lock_acquire+0x1a7/0x4e0
[   70.063114][    C2]  ? hrtimer_interrupt+0x100/0x810
[   70.064133][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   70.065138][    C2]  ? hrtimer_interrupt+0x339/0x810
[   70.066153][    C2]  ? kvm_clock_read+0x14/0x30
[   70.067127][    C2]  _raw_spin_lock_irqsave+0x47/0x70
[   70.068136][    C2]  ? hrtimer_interrupt+0x100/0x810
[   70.069135][    C2]  hrtimer_interrupt+0x100/0x810
[   70.070114][    C2]  ? __pfx_sched_clock_cpu+0x10/0x10
[   70.071129][    C2]  __sysvec_apic_timer_interrupt+0x146/0x3f0
[   70.072201][    C2]  sysvec_apic_timer_interrupt+0x8a/0xb0
[   70.073246][    C2]  </IRQ>
[   70.074097][    C2]  <TASK>
[   70.074941][    C2]  asm_sysvec_apic_timer_interrupt+0x16/0x20
[   70.076012][    C2] RIP: 0010:lockdep_unregister_key+0x105/0x250
[   70.077089][    C2] Code: 7c 08 84 d2 0f 85 29 01 00 00 8b 05 75 7e 19 04 85 c0 74 02 0f 0b e8 6a e7f
[   70.079842][    C2] RSP: 0018:ffff88810ad67ca0 EFLAGS: 00000206
[   70.080983][    C2] RAX: 0000000000000002 RBX: ffffffff897afb78 RCX: 0000000000000001
[   70.082242][    C2] RDX: 0000000000000000 RSI: ffffffff85ce0fa0 RDI: ffffffff85fa7100
[   70.083498][    C2] RBP: ffff88812da24ae0 R08: 0000000000000000 R09: ffff8883aef45c2f
[   70.084767][    C2] R10: ffffed1075de8b85 R11: ffff8881215bb280 R12: 0000000000000000
[   70.086040][    C2] R13: 0000000000000246 R14: ffffffff89939550 R15: ffff88812f5c3118
[   70.087312][    C2]  nvmet_rdma_free_queue+0x2e/0x390 [nvmet_rdma]
[   70.088457][    C2]  nvmet_rdma_release_queue_work+0x3e/0x90 [nvmet_rdma]
[   70.089629][    C2]  process_one_work+0x7e4/0x1390
[   70.090651][    C2]  ? __pfx_lock_acquire+0x10/0x10
[   70.091665][    C2]  ? __pfx_process_one_work+0x10/0x10
[   70.092698][    C2]  ? __pfx_do_raw_spin_lock+0x10/0x10
[   70.093717][    C2]  worker_thread+0xf7/0x12b0
[   70.094656][    C2]  ? __kthread_parkme+0xc1/0x1f0
[   70.095592][    C2]  ? __pfx_worker_thread+0x10/0x10
[   70.096533][    C2]  kthread+0x29e/0x340
[   70.097392][    C2]  ? __pfx_kthread+0x10/0x10
[   70.098286][    C2]  ret_from_fork+0x2c/0x50
[   70.099170][    C2]  </TASK>
[   70.099941][    C2] Modules linked in: siw rdma_rxe ib_uverbs ip6_udp_tunnel udp_tunnel nvmet_rdma ng
[   70.107198][    C2] ---[ end trace 0000000000000000 ]---
[   70.108252][    C2] RIP: 0010:__lock_acquire+0x2481/0x5eb0
[   70.109319][    C2] Code: 0f 83 a7 03 00 00 48 8d 1c 5b 48 c1 e3 06 48 81 c3 e0 3f 7b 89 48 b8 00 00f
[   70.112125][    C2] RSP: 0018:ffff8883aef09ce8 EFLAGS: 00010002
[   70.113284][    C2] RAX: dffffc0000000000 RBX: 0000000000000000 RCX: 1ffff11075de1370
[   70.114588][    C2] RDX: 0000000000000008 RSI: 0000000000000004 RDI: 0000000000000040
[   70.115899][    C2] RBP: ffff8881215bc108 R08: 0000000000000001 R09: ffff8883aef3084b
[   70.117205][    C2] R10: ffffed1075de6109 R11: ffff8881215bb280 R12: 0000000000000002
[   70.118517][    C2] R13: 0000000000000000 R14: ffffffff8755632c R15: 00000000ffffffff
[   70.119840][    C2] FS:  0000000000000000(0000) GS:ffff8883aef00000(0000) knlGS:0000000000000000
[   70.121226][    C2] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   70.122449][    C2] CR2: 0000563bf892e000 CR3: 000000012eb74000 CR4: 00000000000006e0
[   70.123791][    C2] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[   70.125131][    C2] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[   70.126462][    C2] Kernel panic - not syncing: Fatal exception in interrupt
[   70.127873][    C2] Kernel Offset: 0x2000000 from 0xffffffff81000000 (relocation range: 0xffffffff80)
[   70.129488][    C2] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]---
#5 lore raw
Quoted 10 lines · Shinichiro Kawasaki
[   70.030101][    C2] general protection fault, probably for non-canonical address 0xdffffc0000000008:I
[   70.054774][    C2] Call Trace:
[   70.055795][    C2]  <IRQ>
[   70.062128][    C2]  lock_acquire+0x1a7/0x4e0
[   70.067127][    C2]  _raw_spin_lock_irqsave+0x47/0x70
[   70.069135][    C2]  hrtimer_interrupt+0x100/0x810
[   70.071129][    C2]  __sysvec_apic_timer_interrupt+0x146/0x3f0
[   70.072201][    C2]  sysvec_apic_timer_interrupt+0x8a/0xb0
[   70.073246][    C2]  </IRQ>
[   70.074097][    C2]  <TASK>

This may indicate a use-after-free of the lock class key by the following code from kernel/workqueue.c:

	lock_map_release(&lockdep_map);

I will try to find a solution.

Bart.

#6 lore raw
In reply to Bart Van Assche
Quoted 5 lines · Bart Van Assche

Waiting in nvmet_rdma_queue_connect() for previous teardowns to complete (flush_workqueue(nvmet_wq)) confuses lockdep. Clear up this confusion by by using dynamic lockdep keys instead of static lockdep keys for queue->release_work.

The concept of adding a code to a working driver only because of some confusion of some other SW competent is not clear to me.

Can we fix the lockdep not to get confused or ignore it if its not a real issue ?

#7 lore raw
Quoted 1 line · Max Gurtovoy

Can we fix the lockdep not to get confused

Lockdep uses static keys by default. Associating a single static key with multiple synchronization object instances can cause false positives. This is why I introduced support for dynamic keys in lockdep about four years ago. I don't think that it is possible to make dynamic keys the default without changing all synchronization primitives.

Quoted 1 line

or ignore it if its not a real issue ?

Most lockdep reports indicate a real problem so every lockdep report should be reviewed. Having to review false positive complaints every time blocktests is run is tedious. I think it is much better to suppress false positive complaints (in a safe way) rather than asking everyone who runs blktests to review all false positive lockdep complaints every time blktests is run.

Bart.

#8 lore raw
Quoted 8 lines · Bart Van Assche
Quoted 1 line · Max Gurtovoy

Can we fix the lockdep not to get confused

Lockdep uses static keys by default. Associating a single static key with multiple synchronization object instances can cause false positives. This is why I introduced support for dynamic keys in lockdep about four years ago. I don't think that it is possible to make dynamic keys the default without changing all synchronization primitives.

Quoted 1 line

or ignore it if its not a real issue ?

Most lockdep reports indicate a real problem so every lockdep report should be reviewed. Having to review false positive complaints every time blocktests is run is tedious. I think it is much better to suppress false positive complaints (in a safe way) rather than asking everyone who runs blktests to review all false positive lockdep complaints every time blktests is run.

Yes, I understand the importance of that for sure.
But the concept of "dirtying" the base code only for the sake of the above reason sounds like a bad practice in general.

Maybe for that we can have some mechanism of side patches that one apply before running the blktests ? This will leave the base code clean but will need to maintain these patches to be applied cleanly on the base code. But I guess we can have some automation for that..

Quoted 2 lines

Bart.

#9 lore raw
Quoted 1 line · Max Gurtovoy

But the concept of "dirtying" the base code only for the sake of the above reason sounds like a bad practice in general.

The use of the words "dirtying" and "bad practice" in the above sentence seems completely inappropriate to me.

Quoted 3 lines

Maybe for that we can have some mechanism of side patches that one apply before running the blktests ? This will leave the base code clean but will need to maintain these patches to be applied cleanly on the base code. But I guess we can have some automation for that..

So you are proposing to maintain this patch as an out-of-tree patch? That sounds like a bad idea to me. Anyone who modifies the NVMe code wouldn't see the out-of-tree patch so the out-of-tree patch would get broken quickly.

Bart.

#10 lore raw
Quoted 4 lines · Bart Van Assche
Quoted 1 line · Max Gurtovoy

But the concept of "dirtying" the base code only for the sake of the above reason sounds like a bad practice in general.

The use of the words "dirtying" and "bad practice" in the above sentence seems completely inappropriate to me.

I'm sorry, this was not my intention. This was the reason I put dirtying under " ". To mention explicitly that it's not really dirtying.

Quoted 6 lines
Quoted 4 lines

Maybe for that we can have some mechanism of side patches that one apply before running the blktests ? This will leave the base code clean but will need to maintain these patches to be applied cleanly on the base code. But I guess we can have some automation for that..

So you are proposing to maintain this patch as an out-of-tree patch? That sounds like a bad idea to me. Anyone who modifies the NVMe code wouldn't see the out-of-tree patch so the out-of-tree patch would get broken quickly.

In tree patches that can be applied before building the environment for testing. This way the base code stays clean and you also fix the blktests false-positive issues.

Quoted 3 lines

Bart.

maintainer Reviewed-by
#11 lore raw
In reply to Bart Van Assche
Quoted 8 lines
Quoted 2 lines

Can we fix the lockdep not to get confused

Lockdep uses static keys by default. Associating a single static key with multiple synchronization object instances can cause false positives. This is why I introduced support for dynamic keys in lockdep about four years ago. I don't think that it is possible to make dynamic keys the default without changing all synchronization primitives.

Quoted 1 line

or ignore it if its not a real issue ?

Most lockdep reports indicate a real problem so every lockdep report should be reviewed. Having to review false positive complaints every time blocktests is run is tedious. I think it is much better to suppress false positive complaints (in a safe way) rather than asking everyone who runs blktests to review all false positive lockdep complaints every time blktests is run.

Agree, shutting up just this complaint would be great. This looks perfectly fine to me.

Reviewed-by on v1 1/1
#12 lore raw
In reply to Bart Van Assche
Quoted 94 lines · Bart Van Assche

Waiting in nvmet_rdma_queue_connect() for previous teardowns to complete (flush_workqueue(nvmet_wq)) confuses lockdep. Clear up this confusion by by using dynamic lockdep keys instead of static lockdep keys for queue->release_work.

See also the following report by Shinichiro:
https://lore.kernel.org/all/rsmmxrchy6voi5qhl4irss5sprna3f5owkqtvybxglcv2pnylm@xmrnpfu3tfpe/

This patch avoids that lockdep reports the following:

======================================================
WARNING: possible circular locking dependency detected
4.19.0-dbg+ #1 Not tainted
------------------------------------------------------
kworker/u12:0/7 is trying to acquire lock:
00000000c03a91d1 (&id_priv->handler_mutex){+.+.}, at: rdma_destroy_id+0x6f/0x440 [rdma_cm]

but task is already holding lock:
(work_completion)(&queue->release_work)){+.+.}, at: process_one_work+0x3c9/0x9f0 which lock already depends on the new lock.

the existing dependency chain (in reverse order) is:
-> #3 ((work_completion)(&queue->release_work)){+.+.}:

        process_one_work+0x447/0x9f0
        worker_thread+0x63/0x5a0
        kthread+0x1cf/0x1f0
        ret_from_fork+0x24/0x30
-> #2 ((wq_completion)"nvmet-rdma-delete-wq"){+.+.}:
        flush_workqueue+0xf3/0x970
        nvmet_rdma_cm_handler+0x1320/0x170f [nvmet_rdma]
        cma_ib_req_handler+0x72f/0xf90 [rdma_cm]
        cm_process_work+0x2e/0x110 [ib_cm]
        cm_work_handler+0x431e/0x50ba [ib_cm]
        process_one_work+0x481/0x9f0
        worker_thread+0x63/0x5a0
        kthread+0x1cf/0x1f0
        ret_from_fork+0x24/0x30
-> #1 (&id_priv->handler_mutex/1){+.+.}:
        __mutex_lock+0xfe/0xbe0
        mutex_lock_nested+0x1b/0x20
        cma_ib_req_handler+0x6aa/0xf90 [rdma_cm]
        cm_process_work+0x2e/0x110 [ib_cm]
        cm_work_handler+0x431e/0x50ba [ib_cm]
        process_one_work+0x481/0x9f0
        worker_thread+0x63/0x5a0
        kthread+0x1cf/0x1f0
        ret_from_fork+0x24/0x30
-> #0 (&id_priv->handler_mutex){+.+.}:
        lock_acquire+0xc5/0x200
        __mutex_lock+0xfe/0xbe0
        mutex_lock_nested+0x1b/0x20
        rdma_destroy_id+0x6f/0x440 [rdma_cm]
        nvmet_rdma_release_queue_work+0x8e/0x1b0 [nvmet_rdma]
        process_one_work+0x481/0x9f0
        worker_thread+0x63/0x5a0
        kthread+0x1cf/0x1f0
        ret_from_fork+0x24/0x30
other info that might help us debug this:
Chain exists of:
   &id_priv->handler_mutex --> (wq_completion)"nvmet-rdma-delete-wq" --> (work_completion)(&queue->release_work)

Possible unsafe locking scenario:
        CPU0                    CPU1
        ----                    ----
   lock((work_completion)(&queue->release_work));
                                lock((wq_completion)"nvmet-rdma-delete-wq");
                                lock((work_completion)(&queue->release_work));

lock(&id_priv->handler_mutex);

*** DEADLOCK ***

2 locks held by kworker/u12:0/7:
#0: 00000000272134f2 ((wq_completion)"nvmet-rdma-delete-wq"){+.+.}, at: process_one_work+0x3c9/0x9f0 #1: 0000000090531fcd ((work_completion)(&queue->release_work)){+.+.}, at: process_one_work+0x3c9/0x9f0

                stack backtrace:
CPU: 1 PID: 7 Comm: kworker/u12:0 Not tainted 4.19.0-dbg+ #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014
Workqueue: nvmet-rdma-delete-wq nvmet_rdma_release_queue_work [nvmet_rdma]
Call Trace:
  dump_stack+0x86/0xc5
  print_circular_bug.isra.32+0x20a/0x218
  __lock_acquire+0x1c68/0x1cf0
  lock_acquire+0xc5/0x200
  __mutex_lock+0xfe/0xbe0
  mutex_lock_nested+0x1b/0x20
  rdma_destroy_id+0x6f/0x440 [rdma_cm]
  nvmet_rdma_release_queue_work+0x8e/0x1b0 [nvmet_rdma]
  process_one_work+0x481/0x9f0
  worker_thread+0x63/0x5a0
  kthread+0x1cf/0x1f0
  ret_from_fork+0x24/0x30
Quoted 7 lines
Cc Sagi Grimberg <[email protected]>
Cc Max Gurtovoy <[email protected]>
Cc Hannes Reinecke <[email protected]>
Cc Shin'ichiro Kawasaki <[email protected]>

---

much needed fix to stop these repeated reports to save everyone's time ..

-ck