nvme: set non-mdts limits at I/O queue creation
Subsystems (3)
Chaitanya Kulkarni
maintainerPATCH 1/1 nvme: set non-mdts limits at I/O queue creation
In current implementation we set the non-mdts limits by calling nvme_init_non_mdts_limits() from nvme_init_ctrl_finish(). This also tries to set the limits for the discovery controller which has no I/O queues resulting in the warning message reported by the nvme_log_error() when running blktest nvme/002: -
[ 2005.155946] run blktests nvme/002 at 2022-04-09 16:57:47 [ 2005.192223] loop: module loaded [ 2005.196429] nvmet: adding nsid 1 to subsystem blktests-subsystem-0 [ 2005.200334] nvmet: adding nsid 1 to subsystem blktests-subsystem-1
<------------------------------SNIP---------------------------------->
[ 2008.958108] nvmet: adding nsid 1 to subsystem blktests-subsystem-997 [ 2008.962082] nvmet: adding nsid 1 to subsystem blktests-subsystem-998 [ 2008.966102] nvmet: adding nsid 1 to subsystem blktests-subsystem-999 [ 2008.973132] nvmet: creating discovery controller 1 for subsystem nqn.2014-08.org.nvmexpress.discovery for NQN testhostnqn. *[ 2008.973196] nvme1: Identify(0x6), Invalid Field in Command (sct 0x0 / sc 0x2) MORE DNR* [ 2008.974595] nvme nvme1: new ctrl: "nqn.2014-08.org.nvmexpress.discovery" [ 2009.103248] nvme nvme1: Removing ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery"
Export the nvme_init_non_mdts_limits() and move the same call from nvme_ctrl_finish() to each transport right before each transport calls the nvme_scan_work() from nvme_start_ctrl() path, then proceeds to the ns allocation where these limits max_discard_{segments|sectors} and max_write_zeroes_sectors are actually used in folllwing path :-
A. nvme_scan_work()
...nvme_validate_or_alloc_ns()
nvme_alloc_ns()
nvme_update_ns_info()
nvme_update_disk_info()
nvme_config_discard()
blk_queue_max_write_zeroes_sectors()
1. FC:-nvme_fc_create_association()
call nvme_init_mdts_limits() <---
nvme_start_ctrl()
nvme_scan_queue()
nvme_scan_work()
path to ns alloc & accessing ctrl non mdts limits see above A.
2. PCIe:-nvme_reset_work()
nvme_dev_add()
call nvme_init_mdts_limits() <---
nvme_start_ctrl()
nvme_scan_queue()
nvme_scan_work()
path to ns alloc & accessing ctrl non mdts limits see above A.
3. RDMA :-nvme_rdma_setup_ctrl
nvme_rdma_configure_io_queues
call nvme_init_mdts_limits() <---
nvme_start_ctrl()
nvme_scan_queue()
nvme_scan_work()
path to ns alloc & accessing ctrl non mdts limits see above A.
4. TCP :-nvme_tcp_setup_ctrl
nvme_tcp_configure_io_queues
call nvme_init_mdts_limits() <---
nvme_start_ctrl()
nvme_scan_queue()
nvme_scan_work()
path to ns alloc & accessing ctrl non mdts limits see above A.
--- drivers/nvme/host/core.c | 7 ++----- drivers/nvme/host/fc.c | 4 ++++ drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/pci.c | 7 +++++++ drivers/nvme/host/rdma.c | 4 ++++ drivers/nvme/host/tcp.c | 4 ++++ drivers/nvme/target/loop.c | 4 ++++ 7 files changed, 26 insertions(+), 5 deletions(-)
drivers/nvme/host/core.c +2 -5
@@ -2854,7 +2854,7 @@ static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)return val; } -static int nvme_init_non_mdts_limits (struct nvme_ctrl *ctrl) +int nvme_init_non_mdts_limits (struct nvme_ctrl *ctrl) {struct nvme_command c = { };struct nvme_id_ctrl_nvm *id;
@@ -2905,6 +2905,7 @@ static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)kfree (id);return ret; } +EXPORT_SYMBOL_GPL (nvme_init_non_mdts_limits);static int nvme_init_identify (struct nvme_ctrl *ctrl) {
@@ -3082,10 +3083,6 @@ int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl)if (ret)return ret; - ret =nvme_init_non_mdts_limits (ctrl); -if (ret <0 ) -return ret; - ret =nvme_configure_apst (ctrl);if (ret <0 )return ret;
drivers/nvme/host/fc.c +4 -0
@@ -3168,6 +3168,10 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)*/ if (ctrl->ctrl.queue_count >1 ) { + ret =nvme_init_non_mdts_limits (&ctrl->ctrl); +if (ret <0 ) + goto out_term_aen_ops; +if (!ctrl->ioq_live) ret =nvme_fc_create_io_queues (ctrl);else
drivers/nvme/host/nvme.h +1 -0
@@ -784,6 +784,7 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,long nvme_dev_ioctl (struct file *file,unsigned int cmd,unsigned long arg);int nvme_getgeo (struct block_device *bdev,struct hd_geometry *geo); +int nvme_init_non_mdts_limits (struct nvme_ctrl *ctrl);extern const struct attribute_group *nvme_ns_id_attr_groups[];extern const struct pr_ops nvme_pr_ops;
drivers/nvme/host/pci.c +7 -0
@@ -2526,6 +2526,13 @@ static void nvme_dev_add(struct nvme_dev *dev) {int ret; + ret =nvme_init_non_mdts_limits (&dev->ctrl); +if (ret <0 ) { +dev_warn (dev->ctrl.device, +"reading non mdts limit error: %d\n" , ret); +return ; + } +if (!dev->ctrl.tagset) { dev->tagset.ops = &nvme_mq_ops; dev->tagset.nr_hw_queues = dev->online_queues -1 ;
drivers/nvme/host/rdma.c +4 -0
@@ -971,6 +971,10 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)if (ret)return ret; + ret =nvme_init_non_mdts_limits (&ctrl->ctrl); +if (ret <0 ) + goto out_free_io_queues; +if (new) { ctrl->ctrl.tagset =nvme_rdma_alloc_tagset (&ctrl->ctrl, false);if (IS_ERR (ctrl->ctrl.tagset)) {
drivers/nvme/host/tcp.c +4 -0
@@ -1899,6 +1899,10 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)if (ret)return ret; + ret =nvme_init_non_mdts_limits (ctrl); +if (ret <0 ) + goto out_free_io_queues; +if (new) { ctrl->tagset =nvme_tcp_alloc_tagset (ctrl, false);if (IS_ERR (ctrl->tagset)) {
drivers/nvme/target/loop.c +4 -0
@@ -522,6 +522,10 @@ static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) {int ret; + ret =nvme_init_non_mdts_limits (&ctrl->ctrl); +if (ret <0 ) +return ret; + ret =nvme_loop_init_io_queues (ctrl);if (ret)return ret;
Signature · 1 line
2.29.0
Sagi Grimberg
maintainerQuoted 21 lines · Chaitanya Kulkarni
In current implementation we set the non-mdts limits by calling
nvme_init_non_mdts_limits()fromnvme_init_ctrl_finish(). This also tries to set the limits for the discovery controller which has no I/O queues resulting in the warning message reported by thenvme_log_error()when running blktest nvme/002: -[ 2005.155946] run blktests nvme/002 at 2022-04-09 16:57:47 [ 2005.192223] loop: module loaded [ 2005.196429] nvmet: adding nsid 1 to subsystem blktests-subsystem-0 [ 2005.200334] nvmet: adding nsid 1 to subsystem blktests-subsystem-1<------------------------------SNIP---------------------------------->
[ 2008.958108] nvmet: adding nsid 1 to subsystem blktests-subsystem-997 [ 2008.962082] nvmet: adding nsid 1 to subsystem blktests-subsystem-998 [ 2008.966102] nvmet: adding nsid 1 to subsystem blktests-subsystem-999 [ 2008.973132] nvmet: creating discovery controller 1 for subsystem nqn.2014-08.org.nvmexpress.discovery for NQN testhostnqn. *[ 2008.973196] nvme1: Identify(0x6), Invalid Field in Command (sct 0x0 / sc 0x2) MORE DNR* [ 2008.974595] nvme nvme1: new ctrl: "nqn.2014-08.org.nvmexpress.discovery" [ 2009.103248] nvme nvme1: Removing ctrl: NQN "nqn.2014-08.org.nvmexpress.discovery"
Quoted 5 lines
Export the
nvme_init_non_mdts_limits()and move the same call fromnvme_ctrl_finish()to each transport right before each transport calls thenvme_scan_work()fromnvme_start_ctrl()path, then proceeds to the ns allocation where these limits max_discard_{segments|sectors} andmax_write_zeroes_sectorsare actually used in folllwing path :-
Why not just check the ctrl->queue_count and call it accordingly?
Quoted 28 lines · Sagi Grimberg
Quoted 25 lines · Chaitanya Kulkarni
In current implementation we set the non-mdts limits by calling
nvme_init_non_mdts_limits()fromnvme_init_ctrl_finish(). This also tries to set the limits for the discovery controller which has no I/O queues resulting in the warning message reported by thenvme_log_error()when running blktest nvme/002: -[ 2005.155946] run blktests nvme/002 at 2022-04-09 16:57:47 [ 2005.192223] loop: module loaded [ 2005.196429] nvmet: adding nsid 1 to subsystem blktests-subsystem-0 [ 2005.200334] nvmet: adding nsid 1 to subsystem blktests-subsystem-1<------------------------------SNIP---------------------------------->
[ 2008.958108] nvmet: adding nsid 1 to subsystem blktests-subsystem-997 [ 2008.962082] nvmet: adding nsid 1 to subsystem blktests-subsystem-998 [ 2008.966102] nvmet: adding nsid 1 to subsystem blktests-subsystem-999 [ 2008.973132] nvmet: creating discovery controller 1 for subsystem nqn.2014-08.org.nvmexpress.discovery for NQN testhostnqn. *[ 2008.973196] nvme1: Identify(0x6), Invalid Field in Command (sct 0x0 / sc 0x2) MORE DNR* [ 2008.974595] nvme nvme1: new ctrl: "nqn.2014-08.org.nvmexpress.discovery" [ 2009.103248] nvme nvme1: Removing ctrl: NQN"nqn.2014-08.org.nvmexpress.discovery"
Quoted 7 lines
Quoted 5 lines
Export the
nvme_init_non_mdts_limits()and move the same call fromnvme_ctrl_finish()to each transport right before each transport calls thenvme_scan_work()fromnvme_start_ctrl()path, then proceeds to the ns allocation where these limits max_discard_{segments|sectors} andmax_write_zeroes_sectorsare actually used in folllwing path :-Why not just check the
ctrl->queue_countand call it accordingly?
nvme_init_ctrl_finish() is called before the I/O queues are allocated so queue_count value will be always equal to 1 when checked in the nvme_init_ctrl_finish() before calling the nvme_init_non_mdts_limits() for PCIe :-
nvme_reset_work()
...
nvme_init_ctrl_finish()
nvme_init_non_mdts_limits() <--- ctrl->queue_count = 1
nvme_dev_add()
nvme_setup_io_queues()
nvme_create_io_queues()
nvme_alloc_queue()
dev->ctrl.queue_count++ <---...
Also, this patch groups reading ctrl non-mdts limits right before I/O queue allocation for each transport, since these limits are only used for I/O queue allocation.
-ck
Keith Busch
maintainerQuoted 3 lines · Chaitanya Kulkarni
Also, this patch groups reading ctrl non-mdts limits right before I/O queue allocation for each transport, since these limits are only used for I/O queue allocation.
Maybe just check that 'ctrl->cntrltype == NVME_CTRL_IO' instead?
Quoted 6 lines · Keith Busch
Quoted 3 lines · Chaitanya Kulkarni
Also, this patch groups reading ctrl non-mdts limits right before I/O queue allocation for each transport, since these limits are only used for I/O queue allocation.
Maybe just check that '
ctrl->cntrltype==NVME_CTRL_IO' instead?
I did that initially, but if I remember correctly versions prior to 1.4 don't necessarily report a valid ctrl type.
-ck
Keith Busch
maintainerQuoted 10 lines · Chaitanya Kulkarni
Quoted 6 lines · Keith Busch
Quoted 3 lines · Chaitanya Kulkarni
Also, this patch groups reading ctrl non-mdts limits right before I/O queue allocation for each transport, since these limits are only used for I/O queue allocation.
Maybe just check that '
ctrl->cntrltype==NVME_CTRL_IO' instead?I did that initially, but if I remember correctly versions prior to 1.4 don't necessarily report a valid ctrl type.
I guess we could assume IO controller if < 1.4 since those are the only type that could have existed.
Otherwise, I really don't like having this all be transport specific. It sounds like updating the non_mdts limits could safely be done in scan_work since it requires an IO tagset.
Quoted 12 lines · Keith Busch
Quoted 10 lines · Chaitanya Kulkarni
Quoted 6 lines · Keith Busch
Quoted 3 lines · Chaitanya Kulkarni
Also, this patch groups reading ctrl non-mdts limits right before I/O queue allocation for each transport, since these limits are only used for I/O queue allocation.
Maybe just check that '
ctrl->cntrltype==NVME_CTRL_IO' instead?I did that initially, but if I remember correctly versions prior to 1.4 don't necessarily report a valid ctrl type.
Quoted 6 lines
I guess we could assume IO controller if < 1.4 since those are the only type that could have existed.
Otherwise, I really don't like having this all be transport specific. It sounds like updating the
non_mdtslimits could safely be done inscan_worksince it requires an IO tagset.
Even better to do it in nvme_scan_work, let me read the code again and send out V2 as nvme_scan_work is a common point for all the transports.
-ck