nvmet-loop: avoid using mutex in IO hotpath
Subsystems (2)
PATCH 1/1 nvmet-loop: avoid using mutex in IO hotpath
Using mutex lock in IO hot path causes the kernel BUG sleeping while atomic. Shinichiro[1], first encountered this issue while running blktest nvme/052 shown below:
BUG: sleeping function called from invalid context at kernel/locking/mutex.c:585
in_atomic(): 0, irqs_disabled(): 0, non_block: 0, pid: 996, name: (udev-worker)preempt_count: 0, expected: 0
RCU nest depth: 1, expected: 0
2 locks held by (udev-worker)/996:
#0: ffff8881004570c8 (mapping.invalidate_lock){.+.+}-{3:3}, at: page_cache_ra_unbounded+0x155/0x5c0 #1: ffffffff8607eaa0 (rcu_read_lock){....}-{1:2}, at: blk_mq_flush_plug_list+0xa75/0x1950
CPU: 2 UID: 0 PID: 996 Comm: (udev-worker) Not tainted 6.12.0-rc3+ #339 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.3-2.fc40 04/01/2014 Call Trace: <TASK> dump_stack_lvl+0x6a/0x90 __might_resched.cold+0x1f7/0x23d ? __pfx___might_resched+0x10/0x10 ? vsnprintf+0xdeb/0x18f0 __mutex_lock+0xf4/0x1220 ? nvmet_subsys_nsid_exists+0xb9/0x150 [nvmet] ? __pfx_vsnprintf+0x10/0x10 ? __pfx___mutex_lock+0x10/0x10 ? snprintf+0xa5/0xe0 ? xas_load+0x1ce/0x3f0 ? nvmet_subsys_nsid_exists+0xb9/0x150 [nvmet] nvmet_subsys_nsid_exists+0xb9/0x150 [nvmet] ? __pfx_nvmet_subsys_nsid_exists+0x10/0x10 [nvmet] nvmet_req_find_ns+0x24e/0x300 [nvmet] nvmet_req_init+0x694/0xd40 [nvmet] ? blk_mq_start_request+0x11c/0x750 ? nvme_setup_cmd+0x369/0x990 [nvme_core] nvme_loop_queue_rq+0x2a7/0x7a0 [nvme_loop] ? __pfx___lock_acquire+0x10/0x10 ? __pfx_nvme_loop_queue_rq+0x10/0x10 [nvme_loop] __blk_mq_issue_directly+0xe2/0x1d0 ? __pfx___blk_mq_issue_directly+0x10/0x10 ? blk_mq_request_issue_directly+0xc2/0x140 blk_mq_plug_issue_direct+0x13f/0x630 ? lock_acquire+0x2d/0xc0 ? blk_mq_flush_plug_list+0xa75/0x1950 blk_mq_flush_plug_list+0xa9d/0x1950 ? __pfx_blk_mq_flush_plug_list+0x10/0x10 ? __pfx_mpage_readahead+0x10/0x10 __blk_flush_plug+0x278/0x4d0 ? __pfx___blk_flush_plug+0x10/0x10 ? lock_release+0x460/0x7a0 blk_finish_plug+0x4e/0x90 read_pages+0x51b/0xbc0 ? __pfx_read_pages+0x10/0x10 ? lock_release+0x460/0x7a0 page_cache_ra_unbounded+0x326/0x5c0 force_page_cache_ra+0x1ea/0x2f0 filemap_get_pages+0x59e/0x17b0 ? __pfx_filemap_get_pages+0x10/0x10 ? lock_is_held_type+0xd5/0x130 ? __pfx___might_resched+0x10/0x10 ? find_held_lock+0x2d/0x110 filemap_read+0x317/0xb70 ? up_write+0x1ba/0x510 ? __pfx_filemap_read+0x10/0x10 ? inode_security+0x54/0xf0 ? selinux_file_permission+0x36d/0x420 blkdev_read_iter+0x143/0x3b0 vfs_read+0x6ac/0xa20 ? __pfx_vfs_read+0x10/0x10 ? __pfx_vm_mmap_pgoff+0x10/0x10 ? __pfx___seccomp_filter+0x10/0x10 ksys_read+0xf7/0x1d0 ? __pfx_ksys_read+0x10/0x10 do_syscall_64+0x93/0x180 ? lockdep_hardirqs_on_prepare+0x16d/0x400 ? do_syscall_64+0x9f/0x180 ? lockdep_hardirqs_on+0x78/0x100 ? do_syscall_64+0x9f/0x180 ? lockdep_hardirqs_on_prepare+0x16d/0x400 entry_SYSCALL_64_after_hwframe+0x76/0x7e RIP: 0033:0x7f565bd1ce11 Code: 00 48 8b 15 09 90 0d 00 f7 d8 64 89 02 b8 ff ff ff ff eb bd e8 d0 ad 01 00 f3 0f 1e fa 80 3d 35 12 0e 00 00 74 13 31 c0 0f 05 <48> 3d 00 f0 ff ff 77 4f c3 66 0f 1f 44 00 00 55 48 89 e5 48 83 ec RSP: 002b:00007ffd6e7a20c8 EFLAGS: 00000246 ORIG_RAX: 0000000000000000 RAX: ffffffffffffffda RBX: 0000000000001000 RCX: 00007f565bd1ce11 RDX: 0000000000001000 RSI: 00007f565babb000 RDI: 0000000000000014 RBP: 00007ffd6e7a2130 R08: 00000000ffffffff R09: 0000000000000000 R10: 0000556000bfa610 R11: 0000000000000246 R12: 000000003ffff000 R13: 0000556000bfa5b0 R14: 0000000000000e00 R15: 0000556000c07328 </TASK>
Apparently, the above issue is caused due to using mutex lock while we're in IO hot path. It's a regression caused with commit 505363957fad ("nvmet: fix nvme status code when namespace is disabled"). The mutex ->su_mutex is used to find whether a disabled nsid exists in the config group or not. This is to differentiate between a nsid that is disabled vs non-existent.
To mitigate the above issue, we've worked upon a fix[2] where we now insert nsid in subsys Xarray as soon as it's created under config group and later when that nsid is enabled, we add an Xarray mark on it and set ns->enabled to true. The Xarray mark is useful while we need to loop through all enabled namepsaces under a subsystem using xa_for_each_marked() API. If later a nsid is disabled then we clear Xarray mark from it and also set ns->enabled to false. It's only when nsid is deleted from the config group we delete it from the Xarray.
So with this change, now we could easily differentiate a nsid is disabled (i.e. Xarray entry for ns exists but ns->enabled is set to false) vs non- existent (i.e.Xarray entry for ns doesn't exist).
--- drivers/nvme/target/admin-cmd.c | 13 ++-- drivers/nvme/target/core.c | 108 +++++++++++++++++++------------- drivers/nvme/target/nvmet.h | 1 + drivers/nvme/target/pr.c | 10 +-- 4 files changed, 79 insertions(+), 53 deletions(-)
drivers/nvme/target/admin-cmd.c +9 -4
@@ -139,7 +139,8 @@ static u16 nvmet_get_smart_log_all(struct nvmet_req *req,unsigned long idx; ctrl = req->sq->ctrl; -xa_for_each (&ctrl->subsys->namespaces, idx, ns) { +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {/* we don't have the right data for file backed ns */ if (!ns->bdev)continue ;
@@ -331,9 +332,11 @@ static u32 nvmet_format_ana_group(struct nvmet_req *req, u32 grpid,u32 count =0 ;if (!(req->cmd->get_log_page.lsp &NVME_ANA_LOG_RGO )) { -xa_for_each (&ctrl->subsys->namespaces, idx, ns) +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {if (ns->anagrpid == grpid) desc->nsids[count++] =cpu_to_le32 (ns->nsid); + } } desc->grpid =cpu_to_le32 (grpid);
@@ -771,7 +774,8 @@ static void nvmet_execute_identify_endgrp_list(struct nvmet_req *req) goto out; } -xa_for_each (&ctrl->subsys->namespaces, idx, ns) { +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {if (ns->nsid <= min_endgid)continue ;
@@ -814,7 +818,8 @@ static void nvmet_execute_identify_nslist(struct nvmet_req *req, bool match_css) goto out; } -xa_for_each (&ctrl->subsys->namespaces, idx, ns) { +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {if (ns->nsid <= min_nsid)continue ;if (match_css && req->ns->csi != req->cmd->identify.csi)
drivers/nvme/target/core.c +63 -45
191 lines not on this page · raw message
drivers/nvme/target/nvmet.h +1 -0
@@ -24,6 +24,7 @@#define NVMET_DEFAULT_VS NVME_VS(2, 1, 0) +#define NVMET_NS_ENABLED XA_MARK_1#define NVMET_ASYNC_EVENTS 4#define NVMET_ERROR_LOG_SLOTS 128#define NVMET_NO_ERROR_LOC ((u16)-1)
drivers/nvme/target/pr.c +6 -4
@@ -60,7 +60,8 @@ u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask) goto success; } -xa_for_each (&ctrl->subsys->namespaces, idx, ns) { +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {if (ns->pr.enable)WRITE_ONCE (ns->pr.notify_mask, mask); }
@@ -1057,7 +1058,7 @@ int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl)* nvmet_pr_init_ns(), see more details in nvmet_ns_enable(). * So just check ns->pr.enable. */ -xa_for_each (&subsys->namespaces, idx, ns) { +xa_for_each_marked (&subsys->namespaces, idx, ns,NVMET_NS_ENABLED ) {if (ns->pr.enable) { ret =nvmet_pr_alloc_and_insert_pc_ref (ns, ctrl->cntlid, &ctrl->hostid);
@@ -1068,7 +1069,7 @@ int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl)return 0 ; free_per_ctrl_refs: -xa_for_each (&subsys->namespaces, idx, ns) { +xa_for_each_marked (&subsys->namespaces, idx, ns,NVMET_NS_ENABLED ) {if (ns->pr.enable) { pc_ref =xa_erase (&ns->pr_per_ctrl_refs, ctrl->cntlid);if (pc_ref)
@@ -1088,7 +1089,8 @@ void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl)kfifo_free (&ctrl->pr_log_mgr.log_queue);mutex_destroy (&ctrl->pr_log_mgr.lock); -xa_for_each (&ctrl->subsys->namespaces, idx, ns) { +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {if (ns->pr.enable) { pc_ref =xa_erase (&ns->pr_per_ctrl_refs, ctrl->cntlid);if (pc_ref)
Signature · 1 line
2.45.2
Quoted 113 lines · Nilay Shroff
Using mutex lock in IO hot path causes the kernel BUG sleeping while atomic. Shinichiro[1], first encountered this issue while running blktest nvme/052 shown below:
BUG: sleeping function called from invalid context at kernel/locking/mutex.c:585
in_atomic(): 0,irqs_disabled(): 0,non_block: 0, pid: 996, name: (udev-worker)preempt_count: 0, expected: 0
RCU nest depth: 1, expected: 0
2 locks held by (udev-worker)/996:
#0:ffff8881004570c8(mapping.invalidate_lock){.+.+}-{3:3}, at:page_cache_ra_unbounded+0x155/0x5c0#1:ffffffff8607eaa0(rcu_read_lock){....}-{1:2}, at:blk_mq_flush_plug_list+0xa75/0x1950CPU: 2 UID: 0 PID: 996 Comm: (udev-worker) Not tainted 6.12.0-rc3+ #339 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.3-2.fc40 04/01/2014 Call Trace: <TASK> dump_stack_lvl+0x6a/0x90 __might_resched.cold+0x1f7/0x23d ? __pfx___might_resched+0x10/0x10 ? vsnprintf+0xdeb/0x18f0 __mutex_lock+0xf4/0x1220 ? nvmet_subsys_nsid_exists+0xb9/0x150 [nvmet] ? __pfx_vsnprintf+0x10/0x10 ? __pfx___mutex_lock+0x10/0x10 ? snprintf+0xa5/0xe0 ? xas_load+0x1ce/0x3f0 ? nvmet_subsys_nsid_exists+0xb9/0x150 [nvmet] nvmet_subsys_nsid_exists+0xb9/0x150 [nvmet] ? __pfx_nvmet_subsys_nsid_exists+0x10/0x10 [nvmet] nvmet_req_find_ns+0x24e/0x300 [nvmet] nvmet_req_init+0x694/0xd40 [nvmet] ? blk_mq_start_request+0x11c/0x750 ? nvme_setup_cmd+0x369/0x990 [nvme_core] nvme_loop_queue_rq+0x2a7/0x7a0 [nvme_loop] ? __pfx___lock_acquire+0x10/0x10 ? __pfx_nvme_loop_queue_rq+0x10/0x10 [nvme_loop] __blk_mq_issue_directly+0xe2/0x1d0 ? __pfx___blk_mq_issue_directly+0x10/0x10 ? blk_mq_request_issue_directly+0xc2/0x140 blk_mq_plug_issue_direct+0x13f/0x630 ? lock_acquire+0x2d/0xc0 ? blk_mq_flush_plug_list+0xa75/0x1950 blk_mq_flush_plug_list+0xa9d/0x1950 ? __pfx_blk_mq_flush_plug_list+0x10/0x10 ? __pfx_mpage_readahead+0x10/0x10 __blk_flush_plug+0x278/0x4d0 ? __pfx___blk_flush_plug+0x10/0x10 ? lock_release+0x460/0x7a0 blk_finish_plug+0x4e/0x90 read_pages+0x51b/0xbc0 ? __pfx_read_pages+0x10/0x10 ? lock_release+0x460/0x7a0 page_cache_ra_unbounded+0x326/0x5c0 force_page_cache_ra+0x1ea/0x2f0 filemap_get_pages+0x59e/0x17b0 ? __pfx_filemap_get_pages+0x10/0x10 ? lock_is_held_type+0xd5/0x130 ? __pfx___might_resched+0x10/0x10 ? find_held_lock+0x2d/0x110 filemap_read+0x317/0xb70 ? up_write+0x1ba/0x510 ? __pfx_filemap_read+0x10/0x10 ? inode_security+0x54/0xf0 ? selinux_file_permission+0x36d/0x420 blkdev_read_iter+0x143/0x3b0 vfs_read+0x6ac/0xa20 ? __pfx_vfs_read+0x10/0x10 ? __pfx_vm_mmap_pgoff+0x10/0x10 ? __pfx___seccomp_filter+0x10/0x10 ksys_read+0xf7/0x1d0 ? __pfx_ksys_read+0x10/0x10 do_syscall_64+0x93/0x180 ? lockdep_hardirqs_on_prepare+0x16d/0x400 ? do_syscall_64+0x9f/0x180 ? lockdep_hardirqs_on+0x78/0x100 ? do_syscall_64+0x9f/0x180 ? lockdep_hardirqs_on_prepare+0x16d/0x400 entry_SYSCALL_64_after_hwframe+0x76/0x7e RIP: 0033:0x7f565bd1ce11 Code: 00 48 8b 15 09 90 0d 00 f7 d8 64 89 02 b8 ff ff ff ff eb bd e8 d0 ad 01 00 f3 0f 1e fa 80 3d 35 12 0e 00 00 74 13 31 c0 0f 05 <48> 3d 00 f0 ff ff 77 4f c3 66 0f 1f 44 00 00 55 48 89 e5 48 83 ec RSP: 002b:00007ffd6e7a20c8 EFLAGS: 00000246 ORIG_RAX: 0000000000000000 RAX: ffffffffffffffda RBX: 0000000000001000 RCX: 00007f565bd1ce11 RDX: 0000000000001000 RSI: 00007f565babb000 RDI: 0000000000000014 RBP: 00007ffd6e7a2130 R08: 00000000ffffffff R09: 0000000000000000 R10: 0000556000bfa610 R11: 0000000000000246 R12: 000000003ffff000 R13: 0000556000bfa5b0 R14: 0000000000000e00 R15: 0000556000c07328 </TASK>Apparently, the above issue is caused due to using mutex lock while we're in IO hot path. It's a regression caused with commit
505363957fad("nvmet: fix nvme status code when namespace is disabled"). The mutex ->su_mutex is used to find whether a disabled nsid exists in the config group or not. This is to differentiate between a nsid that is disabled vs non-existent.To mitigate the above issue, we've worked upon a fix[2] where we now insert nsid in subsys Xarray as soon as it's created under config group and later when that nsid is enabled, we add an Xarray mark on it and set
ns->enabledto true. The Xarray mark is useful while we need to loop through all enabled namepsaces under a subsystem usingxa_for_each_marked()API. If later a nsid is disabled then we clear Xarray mark from it and also setns->enabledto false. It's only when nsid is deleted from the config group we delete it from the Xarray.So with this change, now we could easily differentiate a nsid is disabled (i.e. Xarray entry for ns exists but
ns->enabledis set to false) vs non- existent (i.e.Xarray entry for ns doesn't exist).Reported-by Shinichiro Kawasaki <[email protected]>Closes https://lore.kernel.org/linux-nvme/tqcy3sveity7p56v7ywp7ssyviwcb3w4623cnxj3knoobfcanq@yxgt2mjkbkam/ [1] open hereFixes 505363957fad ("nvmet: fix nvme status code when namespace is disabled")Fix-suggested-by Christoph Hellwig <[email protected]>Signed-off-by Nilay Shroff <[email protected]>
Quoted 7 lines
--- drivers/nvme/target/admin-cmd.c | 13 ++-- drivers/nvme/target/core.c | 108 +++++++++++++++++++------------- drivers/nvme/target/nvmet.h | 1 + drivers/nvme/target/pr.c | 10 +-- 4 files changed, 79 insertions(+), 53 deletions(-)
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Quoted 3 lines · Nilay Shroff
Using mutex lock in IO hot path causes the kernel BUG sleeping while atomic. Shinichiro[1], first encountered this issue while running blktest nvme/052 shown below:
Looks good.
-ck
Christoph Hellwig
maintainerdrivers/nvme/target/admin-cmd.c static u16 nvmet_get_smart_log_all(struct nvmet_req *req, · line 143
Quoted 3 lines · Nilay Shroff
drivers/nvme/target/admin-cmd.c +2
-1-xa_for_each (&ctrl->subsys->namespaces, idx, ns) { +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {
Given that this is duplicated quite a bit, maybe add a nvmet_for_each_enabled_ns wrapper for it? And also a nvmet_for_each_ns one that iterates all namespaces?
Otherwise this looks good, and I really like the use of the xarray mark to simplify things!
Quoted 8 lines · Christoph Hellwig
drivers/nvme/target/admin-cmd.c static u16 nvmet_get_smart_log_all(struct nvmet_req *req, · line 143
Quoted 3 lines · Nilay Shroff
drivers/nvme/target/admin-cmd.c +2
-1-xa_for_each (&ctrl->subsys->namespaces, idx, ns) { +xa_for_each_marked (&ctrl->subsys->namespaces, idx, + ns,NVMET_NS_ENABLED ) {Given that this is duplicated quite a bit, maybe add a
nvmet_for_each_enabled_nswrapper for it? And also anvmet_for_each_nsone that iterates all namespaces?
Yeah sure, I will add those wrappers and spin a new patch.
Quoted 3 lines
Otherwise this looks good, and I really like the use of the xarray mark to simplify things!
Thanks,
--Nilay