nvme-fabrics: introduce existing_ctrl operation
PATCH 1/3 nvme-fabrics: introduce existing_ctrl operation
The check if the connection request matches an existing controller is duplicated in several transports (such as RDMA and TCP). Move it to common code as optional operation.
Also, this check can be done before creating the controller to simplify the flow.
--- drivers/nvme/host/fabrics.c | 7 +++++++ drivers/nvme/host/fabrics.h | 3 +++ 2 files changed, 10 insertions(+)
drivers/nvme/host/fabrics.c +7 -0
@@ -1100,6 +1100,13 @@ nvmf_create_ctrl(struct device *dev, const char *buf)if (ret) goto out_module_put; +if (ops->existing_ctrl) { +if (!opts->duplicate_connect && ops->existing_ctrl (opts)) { + ret = -EALREADY ; + goto out_module_put; + } + } + ctrl = ops->create_ctrl (dev, opts);if (IS_ERR (ctrl)) { ret =PTR_ERR (ctrl);
drivers/nvme/host/fabrics.h +3 -0
@@ -153,6 +153,8 @@ struct nvmf_ctrl_options {* that would go into starting up that fabric * for the purpose of conneciton to an NVMe controller * using that fabric technology. +* @existing_ctrl(): optional function pointer that checks if a connection +* request matches an existing controller. * * Notes: * 1. At minimum, 'required_opts' and 'allowed_opts' should
@@ -170,6 +172,7 @@ struct nvmf_transport_ops {int allowed_opts;struct nvme_ctrl *(*create_ctrl)(struct device *dev,struct nvmf_ctrl_options *opts); +bool (*existing_ctrl)(struct nvmf_ctrl_options *opts); }; static inline bool
Signature · 1 line
2.18.1
PATCH 2/3 nvme-rdma: implement existing_ctrl operation
The logic of checking matching connections moved to the core layer. Implement the needed optional operation and remove the common logic.
--- drivers/nvme/host/rdma.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
drivers/nvme/host/rdma.c +1 -5
@@ -2326,11 +2326,6 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, } } -if (!opts->duplicate_connect &&nvme_rdma_existing_controller (opts)) { - ret = -EALREADY ; - goto out_free_ctrl; - } -INIT_DELAYED_WORK (&ctrl->reconnect_work, nvme_rdma_reconnect_ctrl_work);INIT_WORK (&ctrl->err_work, nvme_rdma_error_recovery_work);
@@ -2390,6 +2385,7 @@ static struct nvmf_transport_ops nvme_rdma_transport = {NVMF_OPT_NR_WRITE_QUEUES |NVMF_OPT_NR_POLL_QUEUES | NVMF_OPT_TOS, .create_ctrl = nvme_rdma_create_ctrl, + .existing_ctrl = nvme_rdma_existing_controller, };static void nvme_rdma_remove_one (struct ib_device *ib_device,void *client_data)
Signature · 1 line
2.18.1
PATCH 3/3 nvme-tcp: implement existing_ctrl operation
In reply to Max GurtovoyThe logic of checking matching connections moved to the core layer. Implement the needed optional operation and remove the common logic.
--- drivers/nvme/host/tcp.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)
drivers/nvme/host/tcp.c +1 -5
@@ -2608,11 +2608,6 @@ static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, } } -if (!opts->duplicate_connect &&nvme_tcp_existing_controller (opts)) { - ret = -EALREADY ; - goto out_free_ctrl; - } - ctrl->queues =kcalloc (ctrl->ctrl.queue_count,sizeof (*ctrl->queues),GFP_KERNEL );if (!ctrl->queues) {
@@ -2666,6 +2661,7 @@ static struct nvmf_transport_ops nvme_tcp_transport = {NVMF_OPT_NR_WRITE_QUEUES |NVMF_OPT_NR_POLL_QUEUES | NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE, .create_ctrl = nvme_tcp_create_ctrl, + .existing_ctrl = nvme_tcp_existing_controller, };static int __init nvme_tcp_init_module (void )
Signature · 1 line
2.18.1
Quoted 45 lines · Max Gurtovoy
The check if the connection request matches an existing controller is duplicated in several transports (such as RDMA and TCP). Move it to common code as optional operation.
Also, this check can be done before creating the controller to simplify the flow.
Reviewed-by Israel Rukshin <[email protected]>Signed-off-by Max Gurtovoy <[email protected]>--- drivers/nvme/host/fabrics.c | 7 +++++++ drivers/nvme/host/fabrics.h | 3 +++ 2 files changed, 10 insertions(+)drivers/nvme/host/fabrics.c +7
-0@@ -1100,6 +1100,13 @@ nvmf_create_ctrl(struct device *dev, const char *buf)if (ret) goto out_module_put; +if (ops->existing_ctrl) { +if (!opts->duplicate_connect && ops->existing_ctrl (opts)) { + ret = -EALREADY ; + goto out_module_put; + } + } + ctrl = ops->create_ctrl (dev, opts);if (IS_ERR (ctrl)) { ret = PTR_ERR(ctrl); > diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.hindex
a6e22116e139..587b92575a9b100644drivers/nvme/host/fabrics.h +2
-0@@ -153,6 +153,8 @@ struct nvmf_ctrl_options {* that would go into starting up that fabric * for the purpose of conneciton to an NVMe controller * using that fabric technology. +* @existing_ctrl(): optional function pointer that checks if a connection +* request matches an existing controller. * * Notes: * 1. At minimum, 'required_opts' and 'allowed_opts' should
drivers/nvme/host/fabrics.h struct nvmf_transport_ops { · line 178
Quoted 7 lines
drivers/nvme/host/fabrics.h +1
-0@@ -170,6 +172,7 @@ struct nvmf_transport_ops {int allowed_opts;struct nvme_ctrl *(*create_ctrl)(struct device *dev,struct nvmf_ctrl_options *opts); +bool (*existing_ctrl)(struct nvmf_ctrl_options *opts); }; static inline bool
A bit disheartening we're "commonizing" in manner that FC can't participate. I looked at FC supporting this, and although it could, it's a bunch of code duplication, more code than leaving rdma/tcp code in w/o optimization, and FC still has to repeat what it does in controller create as there are other checks to be made. It's more optimal to leave things as is on FC.
Max: mind pinging me ahead of time when creating these common items so we can get the 3rd transport participating if possible ?
-- james
Quoted 56 lines · James Smart
Quoted 51 lines · Max Gurtovoy
The check if the connection request matches an existing controller is duplicated in several transports (such as RDMA and TCP). Move it to common code as optional operation.
Also, this check can be done before creating the controller to simplify the flow.
Reviewed-by Israel Rukshin <[email protected]>Signed-off-by Max Gurtovoy <[email protected]>---
drivers/nvme/host/fabrics.c| 7 +++++++
drivers/nvme/host/fabrics.h| 3 +++2 files changed, 10 insertions(+)drivers/nvme/host/fabrics.c +6
-0@@ -1100,6 +1100,13 @@ nvmf_create_ctrl(struct device *dev, const char *buf) if (ret) goto out_module_put; + if (ops->existing_ctrl) { + if (!opts->duplicate_connect && ops->existing_ctrl(opts)) { + ret = -EALREADY; + goto out_module_put; + } + } + ctrl = ops->create_ctrl(dev, opts); if (IS_ERR(ctrl)) {ret =
PTR_ERR(ctrl); > diff --gita/drivers/nvme/host/fabrics.hb/drivers/nvme/host/fabrics.hindexa6e22116e139..587b92575a9b100644drivers/nvme/host/fabrics.h +1
-0@@ -153,6 +153,8 @@ struct nvmf_ctrl_options { * that would go into starting up that fabric * for the purpose of conneciton to an NVMe controller * using that fabric technology. + * @existing_ctrl(): optional function pointer that checks if aconnection
+ * request matches an existing controller. *
* Notes:
* 1. At minimum, 'required_opts' and 'allowed_opts' shoulddrivers/nvme/host/fabrics.h +1
-0@@ -170,6 +172,7 @@ struct nvmf_transport_ops { int allowed_opts; struct nvme_ctrl *(*create_ctrl)(struct device *dev, struct nvmf_ctrl_options *opts); + bool (*existing_ctrl)(struct nvmf_ctrl_options *opts); };Quoted 1 line
static inline bool
Reviewed-by James Smart <[email protected]>
Quoted 1 line
A bit disheartening we're "commonizing" in manner that FC can't participate. I looked at FC supporting this, and although it could, it's a bunch of code duplication, more code than leaving rdma/tcp code in w/o optimization, and FC still has to repeat what it does in controller create as there are other checks to be made. It's more optimal to leave things as is on FC.
I tried to make this change to be applicable to FC as well but I wasn't able to figure it out so I didn't add changes there.
As you mentioned, it's better to leave things there as is for now.
I don't see any reason not to add the FC as an incremental patch to this series.
Quoted 2 lines
Max: mind pinging me ahead of time when creating these common items so we can get the 3rd transport participating if possible ?
Sure.
I think that a lot of logic can be moved from the transports to the core and I did an attempt in the past to do so.
But we need to make some preparations in the FC code so it will be able to enjoy it as well.
Quoted 3 lines
-- james
drivers/nvme/host/fabrics.c nvmf_create_ctrl(struct device *dev, const char *buf) · line 1104
Quoted 2 lines
drivers/nvme/host/fabrics.c +2
-0+if (ops->existing_ctrl) { +if (!opts->duplicate_connect && ops->existing_ctrl (opts)) {
Nitpick, but the logic would read easier at least to me as:
if (!opts->duplicate_connect &&
ops->existing_ctrl && ops->existing_ctrl(opts)) {I can fix this up if we get some more positive reviews and this gets applied.
Quoted 9 lines · Max Gurtovoy
The check if the connection request matches an existing controller is duplicated in several transports (such as RDMA and TCP). Move it to common code as optional operation.
Also, this check can be done before creating the controller to simplify the flow.
Reviewed-by Israel Rukshin <[email protected]>Signed-off-by Max Gurtovoy <[email protected]>
The series looks good to me, did you run the blocktests for tcp and rdma for this ?
-ck
Sagi Grimberg
maintainerQuoted 47 lines
The check if the connection request matches an existing controller is duplicated in several transports (such as RDMA and TCP). Move it to common code as optional operation.
Also, this check can be done before creating the controller to simplify the flow.
Reviewed-by Israel Rukshin <[email protected]>Signed-off-by Max Gurtovoy <[email protected]>--- drivers/nvme/host/fabrics.c | 7 +++++++ drivers/nvme/host/fabrics.h | 3 +++ 2 files changed, 10 insertions(+)drivers/nvme/host/fabrics.c +7
-0@@ -1100,6 +1100,13 @@ nvmf_create_ctrl(struct device *dev, const char *buf)if (ret) goto out_module_put; +if (ops->existing_ctrl) { +if (!opts->duplicate_connect && ops->existing_ctrl (opts)) { + ret = -EALREADY ; + goto out_module_put; + } + } + ctrl = ops->create_ctrl (dev, opts);if (IS_ERR (ctrl)) { ret =PTR_ERR (ctrl);drivers/nvme/host/fabrics.h +2
-0@@ -153,6 +153,8 @@ struct nvmf_ctrl_options {* that would go into starting up that fabric * for the purpose of conneciton to an NVMe controller * using that fabric technology. +* @existing_ctrl(): optional function pointer that checks if a connection +* request matches an existing controller. * * Notes: * 1. At minimum, 'required_opts' and 'allowed_opts' should
drivers/nvme/host/fabrics.h struct nvmf_transport_ops { · line 178
Quoted 7 lines
drivers/nvme/host/fabrics.h +1
-0@@ -170,6 +172,7 @@ struct nvmf_transport_ops {int allowed_opts;struct nvme_ctrl *(*create_ctrl)(struct device *dev,struct nvmf_ctrl_options *opts); +bool (*existing_ctrl)(struct nvmf_ctrl_options *opts); }; static inline bool
Maybe do nvmf_existing_ctrl() that iterates with device_for_each_child and filters by transport?
This will eliminate the identical code from rdma/tcp and potentially from fc?
Hi Sagi,
Quoted 48 lines · Sagi Grimberg
Quoted 48 lines
The check if the connection request matches an existing controller is duplicated in several transports (such as RDMA and TCP). Move it to common code as optional operation.
Also, this check can be done before creating the controller to simplify the flow.
Reviewed-by Israel Rukshin <[email protected]>Signed-off-by Max Gurtovoy <[email protected]>---
drivers/nvme/host/fabrics.c| 7 +++++++
drivers/nvme/host/fabrics.h| 3 +++2 files changed, 10 insertions(+)drivers/nvme/host/fabrics.c +6
-0@@ -1100,6 +1100,13 @@ nvmf_create_ctrl(struct device *dev, const char *buf) if (ret) goto out_module_put; + if (ops->existing_ctrl) { + if (!opts->duplicate_connect && ops->existing_ctrl(opts)) { + ret = -EALREADY; + goto out_module_put; + } + } + ctrl = ops->create_ctrl(dev, opts); if (IS_ERR(ctrl)) {ret =
PTR_ERR(ctrl);drivers/nvme/host/fabrics.h +1
-0@@ -153,6 +153,8 @@ struct nvmf_ctrl_options { * that would go into starting up that fabric * for the purpose of conneciton to an NVMe controller * using that fabric technology. + * @existing_ctrl(): optional function pointer that checks if aconnection
+ * request matches an existing controller. *
* Notes:
* 1. At minimum, 'required_opts' and 'allowed_opts' shoulddrivers/nvme/host/fabrics.h +0
-0@@ -170,6 +172,7 @@ struct nvmf_transport_ops {
Quoted 9 lines
Quoted 6 lines
drivers/nvme/host/fabrics.h +1
-0int allowed_opts; struct nvme_ctrl *(*create_ctrl)(struct device *dev, struct nvmf_ctrl_options *opts); + bool (*existing_ctrl)(struct nvmf_ctrl_options *opts); };static inline bool
Maybe do
nvmf_existing_ctrl()that iterates withdevice_for_each_childand filters by transport?
do you propose some incremental logic to this series or to change some logic in the existing patches ?
Quoted 2 lines
This will eliminate the identical code from rdma/tcp and potentially from fc?
We agreed that FC will use this common code after some cleanup/changes.
We can't hold improvements to the subsystem if some specific transport has to do some preparations/re-designs to fit common code.
Quoted 60 lines · Max Gurtovoy
Hi Sagi,
Quoted 48 lines · Sagi Grimberg
Quoted 48 lines
The check if the connection request matches an existing controller is duplicated in several transports (such as RDMA and TCP). Move it to common code as optional operation.
Also, this check can be done before creating the controller to simplify the flow.
Reviewed-by Israel Rukshin <[email protected]>Signed-off-by Max Gurtovoy <[email protected]>---
drivers/nvme/host/fabrics.c| 7 +++++++
drivers/nvme/host/fabrics.h| 3 +++2 files changed, 10 insertions(+)drivers/nvme/host/fabrics.c +6
-0@@ -1100,6 +1100,13 @@ nvmf_create_ctrl(struct device *dev, const char *buf) if (ret) goto out_module_put; + if (ops->existing_ctrl) { + if (!opts->duplicate_connect && ops->existing_ctrl(opts)) { + ret = -EALREADY; + goto out_module_put; + } + } + ctrl = ops->create_ctrl(dev, opts); if (IS_ERR(ctrl)) {ret =
PTR_ERR(ctrl);drivers/nvme/host/fabrics.h +1
-0@@ -153,6 +153,8 @@ struct nvmf_ctrl_options { * that would go into starting up that fabric * for the purpose of conneciton to an NVMe controller * using that fabric technology. + * @existing_ctrl(): optional function pointer that checks if aconnection
+ * request matches an existing controller. *
* Notes:
* 1. At minimum, 'required_opts' and 'allowed_opts' shoulddrivers/nvme/host/fabrics.h +0
-0@@ -170,6 +172,7 @@ struct nvmf_transport_ops {Quoted 9 lines
Quoted 6 lines
drivers/nvme/host/fabrics.h +1
-0int allowed_opts; struct nvme_ctrl *(*create_ctrl)(struct device *dev, struct nvmf_ctrl_options *opts); + bool (*existing_ctrl)(struct nvmf_ctrl_options *opts); };static inline bool
Maybe do
nvmf_existing_ctrl()that iterates withdevice_for_each_childand filters by transport?
Quoted 7 lines
do you propose some incremental logic to this series or to change some logic in the existing patches ?
Quoted 2 lines
This will eliminate the identical code from rdma/tcp and potentially from fc?
We agreed that FC will use this common code after some cleanup/changes.
no - we agreed FC is better off the way it is and not using the common routine. Adding the routine would increase code and add locking for no benefit as the same searching has to be repeated in init_ctrl for checks other than opts.
Quoted 3 lines
We can't hold improvements to the subsystem if some specific transport has to do some preparations/re-designs to fit common code.
Only way to make this meaningfully common is to have a different list mechanism to search through existing controllers.
If we can globalize the controllers so there's a list of them off a transport object then doing something like Sagi suggests works, and no need for a callout.
-- james