nvme-fabrics: reject I/O to offline device
PATCH v2 1/1 nvme-fabrics: reject I/O to offline device
Issue Description:
Commands get stuck while Host NVMe controller (TCP or RDMA) is in reconnect state.
NVMe controller enters into reconnect state when it loses connection with the target. It tries to reconnect every 10 seconds (default) until successful reconnection or until reconnect time-out is reached. The default reconnect time out is 10 minutes. This behavior is different than ISCSI where Commands during reconnect state returns with the following error: "rejecting I/O to offline device"
Fix Description:
Following your suggestions:
1. Add a new session parameter called "fast_fail_tmo".
The timeout is measured in seconds from the controller reconnect, any command beyond that timeout is rejected. The new parameter value may be passed during 'connect', and its default value is 30 seconds. A value of 0 means no timeout (in similar to current behavior).
2. Add a controller flag of "failfast_expired".
3. Add dedicated delayed_work that update the "failfast_expired" controller flag.
4. When entering CONNECTING, schedule the delayed_work based on failfast timeout value.
If transition out of CONNECTING, terminate delayed work item and ensure failfast_expired is false. If delayed work item expires: set "failfast_expired" flag to true.
5. Update nvmf_fail_nonready_command() with check the "failfast_expired" controller flag.
branch nvme/for-5.5
---------------------------------
drivers/nvme/host/core.c +37 -0
@@ -93,6 +93,8 @@static void nvme_put_subsystem (struct nvme_subsystem *subsys);static void nvme_remove_invalid_namespaces (struct nvme_ctrl *ctrl,unsigned nsid); +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl); +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl);static void nvme_set_queue_dying (struct nvme_ns *ns) {
@@ -339,6 +341,8 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,case NVME_CTRL_CONNECTING : changed = true;/* FALLTHRU */ +if (old_state ==NVME_CTRL_CONNECTING ) +nvme_stop_failfast_work (ctrl);default :break ; }
@@ -359,6 +363,7 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,case NVME_CTRL_RESETTING : changed = true;/* FALLTHRU */ +nvme_start_failfast_work (ctrl);default :break ; }
@@ -1033,6 +1038,36 @@ void nvme_stop_keep_alive(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_stop_keep_alive); +static void nvme_failfast_work(struct work_struct *work) +{ +struct nvme_ctrl *ctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, failfast_work); + +spin_lock_irq (&ctrl->lock); +if (ctrl->state ==NVME_CTRL_CONNECTING ) { +set_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +dev_info (ctrl->device,"failfast expired set for controller %s\n" , ctrl->opts->subsysnqn); + } +spin_unlock_irq (&ctrl->lock); +} + +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +schedule_delayed_work (&ctrl->failfast_work, ctrl->opts->fail_fast_tmo *HZ ); +} + +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +cancel_delayed_work_sync (&ctrl->failfast_work); +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +} +static int nvme_identify_ctrl (struct nvme_ctrl *dev,struct nvme_id_ctrl **id) {struct nvme_command c = { };
@@ -3979,6 +4014,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl) {nvme_mpath_stop (ctrl);nvme_stop_keep_alive (ctrl); +nvme_stop_failfast_work (ctrl);flush_work (&ctrl->async_event_work);cancel_work_sync (&ctrl->fw_act_work); }
@@ -4043,6 +4079,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,int ret; ctrl->state =NVME_CTRL_NEW ; +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags);spin_lock_init (&ctrl->lock);mutex_init (&ctrl->scan_lock);INIT_LIST_HEAD (&ctrl->namespaces);
@@ -4057,6 +4094,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,init_waitqueue_head (&ctrl->state_wq);INIT_DELAYED_WORK (&ctrl->ka_work, nvme_keep_alive_work); +INIT_DELAYED_WORK (&ctrl->failfast_work, nvme_failfast_work);memset (&ctrl->ka_cmd,0 ,sizeof (ctrl->ka_cmd)); ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
drivers/nvme/host/fabrics.c +20 -3
@@ -549,6 +549,7 @@ blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl, {if (ctrl->state !=NVME_CTRL_DELETING && ctrl->state !=NVME_CTRL_DEAD && + !test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags) && !blk_noretry_request (rq) && !(rq->cmd_flags &REQ_NVME_MPATH ))return BLK_STS_RESOURCE ;
@@ -612,6 +613,7 @@ bool __nvmf_check_ready(struct nvme_ctrl *ctrl, struct request *rq, { NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" }, { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, { NVMF_OPT_TOS, "tos=%d" }, + { NVMF_OPT_FAIL_FAST_TMO, "fail_fast_tmo=%d" }, { NVMF_OPT_ERR, NULL } };
@@ -630,6 +632,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_io_queues =num_online_cpus (); opts->reconnect_delay =NVMF_DEF_RECONNECT_DELAY ; opts->kato =NVME_DEFAULT_KATO ; + opts->fail_fast_tmo =NVMF_DEF_FAIL_FAST_TMO ; opts->duplicate_connect = false; opts->hdr_digest = false; opts->data_digest = false;
@@ -751,6 +754,17 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,pr_warn ("ctrl_loss_tmo < 0 will reconnect forever\n" ); ctrl_loss_tmo = token;break ; +case NVMF_OPT_FAIL_FAST_TMO : +if (match_int (args, &token)) { + ret = -EINVAL ; + goto out; + } + +if (token) +pr_warn ("fail_fast_tmo != 0, I/O will failed on reconnect controller after %d sec\n" , token); + + opts->fail_fast_tmo = token; +break ;case NVMF_OPT_HOSTNQN :if (opts->host) { pr_err("hostnqn already user-assigned: %s\n",
@@ -881,11 +895,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_poll_queues =0 ; opts->duplicate_connect = true; } -if (ctrl_loss_tmo <0 ) +if (ctrl_loss_tmo <0 ){ opts->max_reconnects =-1 ; -else + }else { opts->max_reconnects =DIV_ROUND_UP (ctrl_loss_tmo, opts->reconnect_delay); +if (ctrl_loss_tmo < opts->fail_fast_tmo) +pr_warn ("failfast tmo (%d) larger than controller loss tmo (%d)\n" , opts->fail_fast_tmo, ctrl_loss_tmo); + }if (!opts->host) {kref_get (&nvmf_default_host->ref);
@@ -985,7 +1002,7 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts)#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \ NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \ NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\ - NVMF_OPT_DISABLE_SQFLOW) + NVMF_OPT_DISABLE_SQFLOW|NVMF_OPT_FAIL_FAST_TMO)
static struct nvme_ctrl * nvmf_create_ctrl(struct device *dev, const char *buf)
drivers/nvme/host/fabrics.h +0 -0
@@ -15,6 +15,8 @@#define NVMF_DEF_RECONNECT_DELAY 10
/* default to 600 seconds of reconnect attempts before giving up */
#define NVMF_DEF_CTRL_LOSS_TMO 600
+/* default to 0: don't fail fast IO commands */
+#define NVMF_DEF_FAIL_FAST_TMO 0
/*
* Define a host as seen by the target. We allocate one at boot, but also
@@ -56,6 +58,7 @@ enum {
NVMF_OPT_NR_WRITE_QUEUES = 1 << 17, NVMF_OPT_NR_POLL_QUEUES = 1 << 18,
NVMF_OPT_TOS = 1 << 19,
+ NVMF_OPT_FAIL_FAST_TMO = 1 << 20,
};
/**
@@ -89,6 +92,7 @@ enum {
* @nr_write_queues: number of queues for write I/O
* @nr_poll_queues: number of queues for polling I/O
* @tos: type of service
+ * @fast_fail_tmo: Fast I/O fail timeout in seconds;
*/
struct nvmf_ctrl_options {
unsigned mask;
@@ -111,6 +115,7 @@ struct nvmf_ctrl_options {
unsigned int nr_write_queues;
unsigned int nr_poll_queues;
int tos;
+ unsigned int fail_fast_tmo;
};
/*
drivers/nvme/host/nvme.h +3 -0
@@ -256,6 +256,7 @@ struct nvme_ctrl {struct work_struct scan_work;struct work_struct async_event_work;struct delayed_work ka_work; +struct delayed_work failfast_work;struct nvme_command ka_cmd;struct work_struct fw_act_work;unsigned long events;
@@ -289,6 +290,8 @@ struct nvme_ctrl {u16 icdoff;u16 maxcmd;int nr_reconnects; +unsigned long flags; +#define NVME_CTRL_FAILFAST_EXPIRED 0struct nvmf_ctrl_options *opts;struct page *discard_page;
-----------------
Regards,
Victor
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 226 lines · Victor Gladkov
Issue Description:
Commands get stuck while Host NVMe controller (TCP or RDMA) is in reconnect state.
NVMe controller enters into reconnect state when it loses connection with the target. It tries to reconnect every 10 seconds (default) until successful reconnection or until reconnect time-out is reached. The default reconnect time out is 10 minutes. This behavior is different than ISCSI where Commands during reconnect state returns with the following error: "rejecting I/O to offline device"Fix Description:
Following your suggestions:
1. Add a new session parameter called "fast_fail_tmo".
The timeout is measured in seconds from the controller reconnect, any command beyond that timeout is rejected. The new parameter value may be passed during 'connect', and its default value is 30 seconds. A value of 0 means no timeout (in similar to current behavior).
2. Add a controller flag of "failfast_expired".
3. Add dedicateddelayed_workthat update the "failfast_expired" controller flag.
4. When entering CONNECTING, schedule thedelayed_workbased on failfast timeout value.
If transition out of CONNECTING, terminate delayed work item and ensurefailfast_expiredis false. If delayed work item expires: set "failfast_expired" flag to true.
5. Updatenvmf_fail_nonready_command()with check the "failfast_expired" controller flag.branch nvme/for-5.5
---------------------------------drivers/nvme/host/core.c +37
-0@@ -93,6 +93,8 @@static void nvme_put_subsystem (struct nvme_subsystem *subsys);static void nvme_remove_invalid_namespaces (struct nvme_ctrl *ctrl,unsigned nsid); +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl); +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl);static void nvme_set_queue_dying (struct nvme_ns *ns) {@@ -339,6 +341,8 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,case NVME_CTRL_CONNECTING : changed = true;/* FALLTHRU */ +if (old_state ==NVME_CTRL_CONNECTING ) +nvme_stop_failfast_work (ctrl);default :break ; }@@ -359,6 +363,7 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,case NVME_CTRL_RESETTING : changed = true;/* FALLTHRU */ +nvme_start_failfast_work (ctrl);default :break ; }@@ -1033,6 +1038,36 @@ void nvme_stop_keep_alive(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_stop_keep_alive); +static void nvme_failfast_work(struct work_struct *work) +{ +struct nvme_ctrl *ctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, failfast_work); + +spin_lock_irq (&ctrl->lock); +if (ctrl->state ==NVME_CTRL_CONNECTING ) { +set_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +dev_info (ctrl->device,"failfast expired set for controller %s\n" , ctrl->opts->subsysnqn); + } +spin_unlock_irq (&ctrl->lock); +} + +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +schedule_delayed_work (&ctrl->failfast_work, ctrl->opts->fail_fast_tmo *HZ ); +} + +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +cancel_delayed_work_sync (&ctrl->failfast_work); +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +} +static int nvme_identify_ctrl (struct nvme_ctrl *dev,struct nvme_id_ctrl **id) {struct nvme_command c = { };@@ -3979,6 +4014,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl) {nvme_mpath_stop (ctrl);nvme_stop_keep_alive (ctrl); +nvme_stop_failfast_work (ctrl);flush_work (&ctrl->async_event_work);cancel_work_sync (&ctrl->fw_act_work); }@@ -4043,6 +4079,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,int ret; ctrl->state =NVME_CTRL_NEW ; +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags);spin_lock_init (&ctrl->lock);mutex_init (&ctrl->scan_lock);INIT_LIST_HEAD (&ctrl->namespaces);@@ -4057,6 +4094,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,init_waitqueue_head (&ctrl->state_wq);INIT_DELAYED_WORK (&ctrl->ka_work, nvme_keep_alive_work); +INIT_DELAYED_WORK (&ctrl->failfast_work, nvme_failfast_work);memset (&ctrl->ka_cmd,0 ,sizeof (ctrl->ka_cmd)); ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;drivers/nvme/host/fabrics.c +20
-3@@ -549,6 +549,7 @@ blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl, {if (ctrl->state !=NVME_CTRL_DELETING && ctrl->state !=NVME_CTRL_DEAD && + !test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags) && !blk_noretry_request (rq) && !(rq->cmd_flags &REQ_NVME_MPATH ))return BLK_STS_RESOURCE ;@@ -612,6 +613,7 @@ bool __nvmf_check_ready(struct nvme_ctrl *ctrl, struct request *rq, { NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" }, { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, { NVMF_OPT_TOS, "tos=%d" }, + { NVMF_OPT_FAIL_FAST_TMO, "fail_fast_tmo=%d" }, { NVMF_OPT_ERR, NULL } };@@ -630,6 +632,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_io_queues =num_online_cpus (); opts->reconnect_delay =NVMF_DEF_RECONNECT_DELAY ; opts->kato =NVME_DEFAULT_KATO ; + opts->fail_fast_tmo =NVMF_DEF_FAIL_FAST_TMO ; opts->duplicate_connect = false; opts->hdr_digest = false; opts->data_digest = false;@@ -751,6 +754,17 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,pr_warn ("ctrl_loss_tmo < 0 will reconnect forever\n" ); ctrl_loss_tmo = token;break ; +case NVMF_OPT_FAIL_FAST_TMO : +if (match_int (args, &token)) { + ret = -EINVAL ; + goto out; + } + +if (token) +pr_warn ("fail_fast_tmo != 0, I/O will failed on reconnect controller after %d sec\n" , token); + + opts->fail_fast_tmo = token; +break ;case NVMF_OPT_HOSTNQN :if (opts->host) { pr_err("hostnqn already user-assigned: %s\n",@@ -881,11 +895,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_poll_queues =0 ; opts->duplicate_connect = true; } -if (ctrl_loss_tmo <0 ) +if (ctrl_loss_tmo <0 ){ opts->max_reconnects =-1 ; -else + }else { opts->max_reconnects =DIV_ROUND_UP (ctrl_loss_tmo, opts->reconnect_delay); +if (ctrl_loss_tmo < opts->fail_fast_tmo) +pr_warn ("failfast tmo (%d) larger than controller loss tmo (%d)\n" , opts->fail_fast_tmo, ctrl_loss_tmo); + }if (!opts->host) {kref_get (&nvmf_default_host->ref);@@ -985,7 +1002,7 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts)#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \ NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \ NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\ - NVMF_OPT_DISABLE_SQFLOW) + NVMF_OPT_DISABLE_SQFLOW|NVMF_OPT_FAIL_FAST_TMO)static struct
nvme_ctrl*nvmf_create_ctrl(struct device *dev, const char *buf)drivers/nvme/host/fabrics.h +5
-0@@ -15,6 +15,8 @@#define NVMF_DEF_RECONNECT_DELAY 10/* default to 600 seconds of reconnect attempts before giving up */ #define NVMF_DEF_CTRL_LOSS_TMO 600 +/* default to 0: don't fail fast IO commands */ +#define NVMF_DEF_FAIL_FAST_TMO 0/* * Define a host as seen by the target. We allocate one at boot, but also @@ -56,6 +58,7 @@ enum {NVMF_OPT_NR_WRITE_QUEUES =1 <<17 ,NVMF_OPT_NR_POLL_QUEUES =1 <<18 ,NVMF_OPT_TOS =1 <<19 , + NVMF_OPT_FAIL_FAST_TMO = 1 << 20, };/** @@ -89,6 +92,7 @@ enum {* @nr_write_queues: number of queues for write I/O * @nr_poll_queues: number of queues for polling I/O * @tos: type of service +* @fast_fail_tmo: Fast I/O fail timeout in seconds; */ struct nvmf_ctrl_options {unsigned mask;@@ -111,6 +115,7 @@ struct nvmf_ctrl_options {unsigned int nr_write_queues;unsigned int nr_poll_queues;int tos; +unsigned int fail_fast_tmo; };/* drivers/nvme/host/nvme.h +1
-0@@ -256,6 +256,7 @@ struct nvme_ctrl {struct work_struct scan_work;struct work_struct async_event_work;struct delayed_work ka_work; +struct delayed_work failfast_work;struct nvme_command ka_cmd;struct work_struct fw_act_work;unsigned long events;
drivers/nvme/host/nvme.h struct nvme_ctrl { · line 297
Quoted 8 lines
drivers/nvme/host/nvme.h +2
-0@@ -289,6 +290,8 @@ struct nvme_ctrl {u16 icdoff;u16 maxcmd;int nr_reconnects; +unsigned long flags; +#define NVME_CTRL_FAILFAST_EXPIRED 0struct nvmf_ctrl_options *opts;struct page *discard_page;
In principle it looks good, but:
What happens if the controller reconnects _after_ failfast has triggered? From my reading it would simply establish the connection, so I/O _could_ continue to be served from this path.
But seeing that I/O has already been failed due to failfast tmo the system will most likely have taken corrective action, such as failing over to another path or informing the cluster manager etc.
So who's going to inform the upper layers that the path has become live again?
Cheers,
Hannes
Signature · 9 lines
Dr. Hannes Reinecke Teamlead Storage & Networking [email protected] +49 911 74053 688 SUSE Software Solutions Germany GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), GF: Felix Imendörffer _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 33 lines · Victor Gladkov
Issue Description:
Commands get stuck while Host NVMe controller (TCP or RDMA) is in reconnect state.
NVMe controller enters into reconnect state when it loses connection with the target. It tries to reconnect every 10 seconds (default) until successful reconnection or until reconnect time-out is reached. The default reconnect time out is 10 minutes. This behavior is different than ISCSI where Commands during reconnect state returns with the following error: "rejecting I/O to offline device"Fix Description:
Following your suggestions:
1. Add a new session parameter called "fast_fail_tmo".
The timeout is measured in seconds from the controller reconnect, any command beyond that timeout is rejected. The new parameter value may be passed during 'connect', and its default value is 30 seconds. A value of 0 means no timeout (in similar to current behavior).
2. Add a controller flag of "failfast_expired".
3. Add dedicateddelayed_workthat update the "failfast_expired" controller flag.
4. When entering CONNECTING, schedule thedelayed_workbased on failfast timeout value.
If transition out of CONNECTING, terminate delayed work item and ensurefailfast_expiredis false. If delayed work item expires: set "failfast_expired" flag to true.
5. Updatenvmf_fail_nonready_command()with check the "failfast_expired" controller flag.branch nvme/for-5.5
---------------------------------drivers/nvme/host/core.c +2
-0@@ -93,6 +93,8 @@static void nvme_put_subsystem (struct nvme_subsystem *subsys);static void nvme_remove_invalid_namespaces (struct nvme_ctrl *ctrl,unsigned nsid); +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl); +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl);static void nvme_set_queue_dying (struct nvme_ns *ns) {
drivers/nvme/host/core.c bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, · line 345
Quoted 5 lines
drivers/nvme/host/core.c +2
-0@@ -339,6 +341,8 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,case NVME_CTRL_CONNECTING : changed = true;/* FALLTHRU */ +if (old_state ==NVME_CTRL_CONNECTING ) +nvme_stop_failfast_work (ctrl);
What you have fine, but a simple reorg to avoid the iftest works fine too:
e.g.
@@ -334,9 +334,11 @@ bool nvme_change_ctrl_state(struct nvme_ switch (new_state) {
case NVME_CTRL_LIVE:
switch (old_state) {
+ case NVME_CTRL_CONNECTING:
+ nvme_stop_failfast_work(ctrl);
+ /* FALLTHRU */
case NVME_CTRL_NEW:
case NVME_CTRL_RESETTING:
- case NVME_CTRL_CONNECTING:
changed = true;
/* FALLTHRU */
default:
Quoted 34 lines
drivers/nvme/host/core.c +20
-0default :break ; }@@ -359,6 +363,7 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,case NVME_CTRL_RESETTING : changed = true;/* FALLTHRU */ +nvme_start_failfast_work (ctrl);default :break ; }@@ -1033,6 +1038,36 @@ void nvme_stop_keep_alive(struct nvme_ctrl *ctrl) }EXPORT_SYMBOL_GPL (nvme_stop_keep_alive); +static void nvme_failfast_work(struct work_struct *work) +{ +struct nvme_ctrl *ctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, failfast_work); + +spin_lock_irq (&ctrl->lock); +if (ctrl->state ==NVME_CTRL_CONNECTING ) { +set_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +dev_info (ctrl->device,"failfast expired set for controller %s\n" , ctrl->opts->subsysnqn); + } +spin_unlock_irq (&ctrl->lock); +} + +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +schedule_delayed_work (&ctrl->failfast_work, ctrl->opts->fail_fast_tmo *HZ ); +}
drivers/nvme/host/core.c void nvme_stop_keep_alive(struct nvme_ctrl *ctrl) · line 1069
Quoted 10 lines
drivers/nvme/host/core.c +10
-0+ +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +cancel_delayed_work_sync (&ctrl->failfast_work); +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +} +
Any reason they aren't added higher in the file to avoid the prototypes ?
Quoted 69 lines
drivers/nvme/host/core.c +3
-0static int nvme_identify_ctrl (struct nvme_ctrl *dev,struct nvme_id_ctrl **id) {struct nvme_command c = { };@@ -3979,6 +4014,7 @@ void nvme_stop_ctrl:q(struct nvme_ctrl *ctrl) {nvme_mpath_stop (ctrl);nvme_stop_keep_alive (ctrl); +nvme_stop_failfast_work (ctrl);flush_work (&ctrl->async_event_work);cancel_work_sync (&ctrl->fw_act_work); }@@ -4043,6 +4079,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,int ret; ctrl->state =NVME_CTRL_NEW ; +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags);spin_lock_init (&ctrl->lock);mutex_init (&ctrl->scan_lock);INIT_LIST_HEAD (&ctrl->namespaces);@@ -4057,6 +4094,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,init_waitqueue_head (&ctrl->state_wq);INIT_DELAYED_WORK (&ctrl->ka_work, nvme_keep_alive_work); +INIT_DELAYED_WORK (&ctrl->failfast_work, nvme_failfast_work);memset (&ctrl->ka_cmd,0 ,sizeof (ctrl->ka_cmd)); ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;drivers/nvme/host/fabrics.c +14
-0@@ -549,6 +549,7 @@ blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl, {if (ctrl->state !=NVME_CTRL_DELETING && ctrl->state !=NVME_CTRL_DEAD && + !test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags) && !blk_noretry_request (rq) && !(rq->cmd_flags &REQ_NVME_MPATH ))return BLK_STS_RESOURCE ;@@ -612,6 +613,7 @@ bool __nvmf_check_ready(struct nvme_ctrl *ctrl, struct request *rq, { NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" }, { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, { NVMF_OPT_TOS, "tos=%d" }, + { NVMF_OPT_FAIL_FAST_TMO, "fail_fast_tmo=%d" }, { NVMF_OPT_ERR, NULL } };@@ -630,6 +632,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_io_queues =num_online_cpus (); opts->reconnect_delay =NVMF_DEF_RECONNECT_DELAY ; opts->kato =NVME_DEFAULT_KATO ; + opts->fail_fast_tmo =NVMF_DEF_FAIL_FAST_TMO ; opts->duplicate_connect = false; opts->hdr_digest = false; opts->data_digest = false;@@ -751,6 +754,17 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,pr_warn ("ctrl_loss_tmo < 0 will reconnect forever\n" ); ctrl_loss_tmo = token;break ; +case NVMF_OPT_FAIL_FAST_TMO : +if (match_int (args, &token)) { + ret = -EINVAL ; + goto out; + } + +if (token) +pr_warn ("fail_fast_tmo != 0, I/O will failed on reconnect controller after %d sec\n" , token); + + opts->fail_fast_tmo = token; +break ;case NVMF_OPT_HOSTNQN :if (opts->host) { pr_err("hostnqn already user-assigned: %s\n",
drivers/nvme/host/fabrics.c static int nvmf_parse_options(struct nvmf_ctrl_options *opts, · line 900
Quoted 8 lines
drivers/nvme/host/fabrics.c +2
-2@@ -881,11 +895,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_poll_queues =0 ; opts->duplicate_connect = true; } -if (ctrl_loss_tmo <0 ) +if (ctrl_loss_tmo <0 ){ opts->max_reconnects =-1 ; -else + }else {
style issues - need spaces before uses of { and } e.g. "if (ctrl_loss_tmo < 0) {" and "} else {"
Quoted 9 lines
drivers/nvme/host/fabrics.c +3
-0opts->max_reconnects =DIV_ROUND_UP (ctrl_loss_tmo, opts->reconnect_delay); +if (ctrl_loss_tmo < opts->fail_fast_tmo) +pr_warn ("failfast tmo (%d) larger than controller loss tmo (%d)\n" , opts->fail_fast_tmo, ctrl_loss_tmo); + }if (!opts->host) {kref_get (&nvmf_default_host->ref);
drivers/nvme/host/fabrics.c void nvmf_free_options(struct nvmf_ctrl_options *opts) · line 1005
Quoted 5 lines
drivers/nvme/host/fabrics.c +1
-1@@ -985,7 +1002,7 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts)#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \ NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \ NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\ - NVMF_OPT_DISABLE_SQFLOW) + NVMF_OPT_DISABLE_SQFLOW|NVMF_OPT_FAIL_FAST_TMO)
spaces here too, or line break/continuation
drivers/nvme/host/fabrics.h line 19
Quoted 13 lines
static struct
nvme_ctrl*nvmf_create_ctrl(struct device *dev, const char *buf)drivers/nvme/host/fabrics.h +2
-0@@ -15,6 +15,8 @@#define NVMF_DEF_RECONNECT_DELAY 10/* default to 600 seconds of reconnect attempts before giving up */ #define NVMF_DEF_CTRL_LOSS_TMO 600 +/* default to 0: don't fail fast IO commands */ +#define NVMF_DEF_FAIL_FAST_TMO 0
I can agree with this - default behavior is old behavior - but given we are talking many minutes for old behavior, perhaps we should be making 0 be immediate failure and not preserve the old behavior ?? Thoughts from anyone ?
Quoted 49 lines
drivers/nvme/host/fabrics.h +3
-0/* * Define a host as seen by the target. We allocate one at boot, but also @@ -56,6 +58,7 @@ enum {NVMF_OPT_NR_WRITE_QUEUES =1 <<17 ,NVMF_OPT_NR_POLL_QUEUES =1 <<18 ,NVMF_OPT_TOS =1 <<19 , + NVMF_OPT_FAIL_FAST_TMO = 1 << 20, };/** @@ -89,6 +92,7 @@ enum {* @nr_write_queues: number of queues for write I/O * @nr_poll_queues: number of queues for polling I/O * @tos: type of service +* @fast_fail_tmo: Fast I/O fail timeout in seconds; */ struct nvmf_ctrl_options {unsigned mask;@@ -111,6 +115,7 @@ struct nvmf_ctrl_options {unsigned int nr_write_queues;unsigned int nr_poll_queues;int tos; +unsigned int fail_fast_tmo; };/* drivers/nvme/host/nvme.h +3
-1@@ -256,6 +256,7 @@ struct nvme_ctrl {struct work_struct scan_work;struct work_struct async_event_work;struct delayed_work ka_work; +struct delayed_work failfast_work;struct nvme_command ka_cmd;struct work_struct fw_act_work;unsigned long events;@@ -289,6 +290,8 @@ struct nvme_ctrl {u16 icdoff;u16 maxcmd;int nr_reconnects; +unsigned long flags; +#define NVME_CTRL_FAILFAST_EXPIRED 0struct nvmf_ctrl_options *opts;struct page *discard_page; -----------------
Quoted 3 lines
Regards,
Victor
overall - looks good. I'd like to see what the answer is to what our default action should be (see above, fail_fast = 0) and agree with the multipath comment Hannes had.
-- james
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
1. Added multipath support for this patch.
2. Small refactoring (according to the review)
Quoted 10 lines · James Smart
I can agree with this - default behavior is old behavior - but given we are talking many minutes for old behavior, perhaps we should be making 0 be immediate failure and not preserve the old behavior ?? Thoughts from anyone ?
overall - looks good. I'd like to see what the answer is to what our default action should be (see above,
fail_fast= 0) and agree with the multipath comment Hannes had.
James, please see your comment on Tue Dec 17 10:03:26 PST 2019
http://lists.infradead.org/pipermail/linux-nvme/2019-December/028483.html
Quoted 3 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (fail_fast_tmo < 0) + opts->fail_fast_tmo_ns = -1;would prefer it be set to 0 to mean disabled. [
J.S.]
Quoted 9 lines · Hannes Reinecke
What happens if the controller reconnects _after_ failfast has triggered? From my reading it would simply establish the connection, so I/O _could_ continue to be served from this path.
But seeing that I/O has already been failed due to failfast tmo the system will most likely have taken corrective action, such as failing over to another path or informing the cluster manager etc.
So who's going to inform the upper layers that the path has become live again?
The function nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) schedules a requeue_work for all namespaces on the controller.
See drivers/nvme/host/core.c:
bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
enum nvme_ctrl_state new_state)
{
..............................................................................
if (changed && ctrl->state == NVME_CTRL_LIVE)
nvme_kick_requeue_lists(ctrl);
return changed;
}---------------------------------
---------------------------------
The updated patch:
branch nvme/for-5.5
---------------------------------
drivers/nvme/host/core.c +38 -2
@@ -321,6 +321,37 @@ bool nvme_cancel_request(struct request *req, void *data, bool reserved) }EXPORT_SYMBOL_GPL (nvme_cancel_request); +static void nvme_failfast_work (struct work_struct *work) +{ +struct nvme_ctrl *ctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, failfast_work); + +spin_lock_irq (&ctrl->lock); +if (ctrl->state ==NVME_CTRL_CONNECTING ) { +set_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +dev_info (ctrl->device,"failfast expired set for controller %s\n" , ctrl->opts->subsysnqn); +nvme_kick_requeue_lists (ctrl); + } +spin_unlock_irq (&ctrl->lock); +} + +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +schedule_delayed_work (&ctrl->failfast_work, ctrl->opts->fail_fast_tmo *HZ ); +} + +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +cancel_delayed_work_sync (&ctrl->failfast_work); +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +} +bool nvme_change_ctrl_state (struct nvme_ctrl *ctrl,enum nvme_ctrl_state new_state) {
@@ -334,9 +365,10 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,switch (new_state) {case NVME_CTRL_LIVE :switch (old_state) { +case NVME_CTRL_CONNECTING : +nvme_stop_failfast_work (ctrl);case NVME_CTRL_NEW :case NVME_CTRL_RESETTING : -case NVME_CTRL_CONNECTING : changed = true;/* FALLTHRU */ default :
@@ -355,8 +387,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,break ;case NVME_CTRL_CONNECTING :switch (old_state) { -case NVME_CTRL_NEW :case NVME_CTRL_RESETTING : +nvme_start_failfast_work (ctrl); +case NVME_CTRL_NEW : changed = true;/* FALLTHRU */ default :
@@ -3979,6 +4012,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl) {nvme_mpath_stop (ctrl);nvme_stop_keep_alive (ctrl); +nvme_stop_failfast_work (ctrl);flush_work (&ctrl->async_event_work);cancel_work_sync (&ctrl->fw_act_work); }
@@ -4043,6 +4077,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,int ret; ctrl->state =NVME_CTRL_NEW ; +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags);spin_lock_init (&ctrl->lock);mutex_init (&ctrl->scan_lock);INIT_LIST_HEAD (&ctrl->namespaces);
@@ -4057,6 +4092,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,init_waitqueue_head (&ctrl->state_wq);INIT_DELAYED_WORK (&ctrl->ka_work, nvme_keep_alive_work); +INIT_DELAYED_WORK (&ctrl->failfast_work, nvme_failfast_work);memset (&ctrl->ka_cmd,0 ,sizeof (ctrl->ka_cmd)); ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
drivers/nvme/host/fabrics.c +20 -3
@@ -549,6 +549,7 @@ blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl, {if (ctrl->state !=NVME_CTRL_DELETING && ctrl->state !=NVME_CTRL_DEAD && + !test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags) && !blk_noretry_request (rq) && !(rq->cmd_flags &REQ_NVME_MPATH ))return BLK_STS_RESOURCE ;
@@ -612,6 +613,7 @@ bool __nvmf_check_ready(struct nvme_ctrl *ctrl, struct request *rq, { NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" }, { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, { NVMF_OPT_TOS, "tos=%d" }, + { NVMF_OPT_FAIL_FAST_TMO, "fail_fast_tmo=%d" }, { NVMF_OPT_ERR, NULL } };
@@ -630,6 +632,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_io_queues =num_online_cpus (); opts->reconnect_delay =NVMF_DEF_RECONNECT_DELAY ; opts->kato =NVME_DEFAULT_KATO ; + opts->fail_fast_tmo =NVMF_DEF_FAIL_FAST_TMO ; opts->duplicate_connect = false; opts->hdr_digest = false; opts->data_digest = false;
@@ -751,6 +754,17 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,pr_warn ("ctrl_loss_tmo < 0 will reconnect forever\n" ); ctrl_loss_tmo = token;break ; +case NVMF_OPT_FAIL_FAST_TMO : +if (match_int (args, &token)) { + ret = -EINVAL ; + goto out; + } + +if (token) +pr_warn ("fail_fast_tmo != 0, I/O will failed on reconnect controller after %d sec\n" , token); + + opts->fail_fast_tmo = token; +break ;case NVMF_OPT_HOSTNQN :if (opts->host) { pr_err("hostnqn already user-assigned: %s\n",
@@ -881,11 +895,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_poll_queues =0 ; opts->duplicate_connect = true; } -if (ctrl_loss_tmo <0 ) +if (ctrl_loss_tmo <0 ) { opts->max_reconnects =-1 ; -else + }else { opts->max_reconnects =DIV_ROUND_UP (ctrl_loss_tmo, opts->reconnect_delay); +if (ctrl_loss_tmo < opts->fail_fast_tmo) +pr_warn ("failfast tmo (%d) larger than controller loss tmo (%d)\n" , opts->fail_fast_tmo, ctrl_loss_tmo); + }if (!opts->host) {kref_get (&nvmf_default_host->ref);
@@ -985,7 +1002,7 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts)#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \ NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \ NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\ - NVMF_OPT_DISABLE_SQFLOW) + NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_FAIL_FAST_TMO)static struct nvme_ctrl *nvmf_create_ctrl (struct device *dev,const char *buf)
drivers/nvme/host/fabrics.h +5 -0
@@ -15,6 +15,8 @@#define NVMF_DEF_RECONNECT_DELAY 10/* default to 600 seconds of reconnect attempts before giving up */ #define NVMF_DEF_CTRL_LOSS_TMO 600 +/* default to 30 seconds of fail fast IO commands */ +#define NVMF_DEF_FAIL_FAST_TMO 0/* * Define a host as seen by the target. We allocate one at boot, but also
@@ -56,6 +58,7 @@ enum {NVMF_OPT_NR_WRITE_QUEUES =1 <<17 ,NVMF_OPT_NR_POLL_QUEUES =1 <<18 ,NVMF_OPT_TOS =1 <<19 , + NVMF_OPT_FAIL_FAST_TMO = 1 << 20, };/**
@@ -89,6 +92,7 @@ enum {* @nr_write_queues: number of queues for write I/O * @nr_poll_queues: number of queues for polling I/O * @tos: type of service +* @fast_fail_tmo_ns: Fast I/O fail timeout in nanoseconds; */ struct nvmf_ctrl_options {unsigned mask;
@@ -111,6 +115,7 @@ struct nvmf_ctrl_options {unsigned int nr_write_queues;unsigned int nr_poll_queues;int tos; +unsigned int fail_fast_tmo; };/*
drivers/nvme/host/multipath.c +3 -1
@@ -281,9 +281,11 @@ static bool nvme_available_path(struct nvme_ns_head *head)list_for_each_entry_rcu (ns, &head->list, siblings) {switch (ns->ctrl->state) { +case NVME_CTRL_CONNECTING : +if (test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ns->ctrl->flags)) +break ;case NVME_CTRL_LIVE :case NVME_CTRL_RESETTING : -case NVME_CTRL_CONNECTING :/* fallthru */ return true;default :
drivers/nvme/host/nvme.h +3 -0
@@ -256,6 +256,7 @@ struct nvme_ctrl {struct work_struct scan_work;struct work_struct async_event_work;struct delayed_work ka_work; +struct delayed_work failfast_work;struct nvme_command ka_cmd;struct work_struct fw_act_work;unsigned long events;
@@ -289,6 +290,8 @@ struct nvme_ctrl {u16 icdoff;u16 maxcmd;int nr_reconnects; +unsigned long flags; +#define NVME_CTRL_FAILFAST_EXPIRED 0struct nvmf_ctrl_options *opts;struct page *discard_page;
-----------------
Regards,
Victor
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 2 lines · Victor Gladkov
1. Added multipath support for this patch.
2. Small refactoring (according to the review)
Anyone have any comments on the latest proposed patch?
Regards,
Victor
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 5 lines · Victor Gladkov
Quoted 2 lines · Victor Gladkov
1. Added multipath support for this patch.
2. Small refactoring (according to the review)Anyone have any comments on the latest proposed patch?
Where is the latest patch? I didn't see a repost. To kick off a discussion please just resent it, in proper patch format and with the proper style to make reviewing it easier.
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 10 lines · Christoph Hellwig
Quoted 5 lines · Victor Gladkov
Quoted 2 lines · Victor Gladkov
1. Added multipath support for this patch.
2. Small refactoring (according to the review)Anyone have any comments on the latest proposed patch?
Where is the latest patch? I didn't see a repost. To kick off a discussion please just resent it, in proper patch format and with the proper style to make reviewing it easier.
Patch updated for branch nvme/for-5.6
---------------------------------
drivers/nvme/host/core.c +38 -2
@@ -321,6 +321,37 @@ }EXPORT_SYMBOL_GPL (nvme_cancel_request); +static void nvme_failfast_work (struct work_struct *work) +{ +struct nvme_ctrl *ctrl =container_of (to_delayed_work (work), + struct nvme_ctrl, failfast_work); + +spin_lock_irq (&ctrl->lock); +if (ctrl->state ==NVME_CTRL_CONNECTING ) { +set_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +dev_info (ctrl->device,"failfast expired set for controller %s\n" , ctrl->opts->subsysnqn); +nvme_kick_requeue_lists (ctrl); + } +spin_unlock_irq (&ctrl->lock); +} + +static void nvme_start_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +schedule_delayed_work (&ctrl->failfast_work, ctrl->opts->fail_fast_tmo *HZ ); +} + +static void nvme_stop_failfast_work (struct nvme_ctrl *ctrl) +{ +if (unlikely (ctrl->opts->fail_fast_tmo ==0 )) +return ; + +cancel_delayed_work_sync (&ctrl->failfast_work); +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags); +} +bool nvme_change_ctrl_state (struct nvme_ctrl *ctrl,enum nvme_ctrl_state new_state) {
@@ -334,9 +365,10 @@switch (new_state) {case NVME_CTRL_LIVE :switch (old_state) { +case NVME_CTRL_CONNECTING : +nvme_stop_failfast_work (ctrl);case NVME_CTRL_NEW :case NVME_CTRL_RESETTING : -case NVME_CTRL_CONNECTING : changed = true;/* FALLTHRU */ default :
@@ -355,8 +387,9 @@break ;case NVME_CTRL_CONNECTING :switch (old_state) { -case NVME_CTRL_NEW :case NVME_CTRL_RESETTING : +nvme_start_failfast_work (ctrl); +case NVME_CTRL_NEW : changed = true;/* FALLTHRU */ default :
@@ -3960,6 +3993,7 @@ {nvme_mpath_stop (ctrl);nvme_stop_keep_alive (ctrl); +nvme_stop_failfast_work (ctrl);flush_work (&ctrl->async_event_work);cancel_work_sync (&ctrl->fw_act_work); }
@@ -4024,6 +4058,7 @@int ret; ctrl->state =NVME_CTRL_NEW ; +clear_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags);spin_lock_init (&ctrl->lock);mutex_init (&ctrl->scan_lock);INIT_LIST_HEAD (&ctrl->namespaces);
@@ -4038,6 +4073,7 @@init_waitqueue_head (&ctrl->state_wq);INIT_DELAYED_WORK (&ctrl->ka_work, nvme_keep_alive_work); +INIT_DELAYED_WORK (&ctrl->failfast_work, nvme_failfast_work);memset (&ctrl->ka_cmd,0 ,sizeof (ctrl->ka_cmd)); ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
drivers/nvme/host/fabrics.c +20 -3
@@ -549,6 +549,7 @@ blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl, {if (ctrl->state !=NVME_CTRL_DELETING && ctrl->state !=NVME_CTRL_DEAD && + !test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ctrl->flags) && !blk_noretry_request (rq) && !(rq->cmd_flags &REQ_NVME_MPATH ))return BLK_STS_RESOURCE ;
@@ -612,6 +613,7 @@ bool __nvmf_check_ready(struct nvme_ctrl *ctrl, struct request *rq, { NVMF_OPT_NR_WRITE_QUEUES, "nr_write_queues=%d" }, { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, { NVMF_OPT_TOS, "tos=%d" }, + { NVMF_OPT_FAIL_FAST_TMO, "fail_fast_tmo=%d" }, { NVMF_OPT_ERR, NULL } };
@@ -630,6 +632,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_io_queues =num_online_cpus (); opts->reconnect_delay =NVMF_DEF_RECONNECT_DELAY ; opts->kato =NVME_DEFAULT_KATO ; + opts->fail_fast_tmo =NVMF_DEF_FAIL_FAST_TMO ; opts->duplicate_connect = false; opts->hdr_digest = false; opts->data_digest = false;
@@ -751,6 +754,17 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,pr_warn ("ctrl_loss_tmo < 0 will reconnect forever\n" ); ctrl_loss_tmo = token;break ; +case NVMF_OPT_FAIL_FAST_TMO : +if (match_int (args, &token)) { + ret = -EINVAL ; + goto out; + } + +if (token) +pr_warn ("fail_fast_tmo != 0, I/O will failed on reconnect controller after %d sec\n" , token); + + opts->fail_fast_tmo = token; +break ;case NVMF_OPT_HOSTNQN :if (opts->host) { pr_err("hostnqn already user-assigned: %s\n",
@@ -881,11 +895,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, opts->nr_poll_queues =0 ; opts->duplicate_connect = true; } -if (ctrl_loss_tmo <0 ) +if (ctrl_loss_tmo <0 ) { opts->max_reconnects =-1 ; -else + }else { opts->max_reconnects =DIV_ROUND_UP (ctrl_loss_tmo, opts->reconnect_delay); +if (ctrl_loss_tmo < opts->fail_fast_tmo) +pr_warn ("failfast tmo (%d) larger than controller loss tmo (%d)\n" , opts->fail_fast_tmo, ctrl_loss_tmo); + }if (!opts->host) {kref_get (&nvmf_default_host->ref);
@@ -985,7 +1002,7 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts)#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \ NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \ NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\ - NVMF_OPT_DISABLE_SQFLOW) + NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_FAIL_FAST_TMO)static struct nvme_ctrl *nvmf_create_ctrl (struct device *dev,const char *buf)
drivers/nvme/host/fabrics.h +5 -0
@@ -15,6 +15,8 @@#define NVMF_DEF_RECONNECT_DELAY 10/* default to 600 seconds of reconnect attempts before giving up */ #define NVMF_DEF_CTRL_LOSS_TMO 600 +/* default to 30 seconds of fail fast IO commands */ +#define NVMF_DEF_FAIL_FAST_TMO 0/* * Define a host as seen by the target. We allocate one at boot, but also
@@ -56,6 +58,7 @@ enum {NVMF_OPT_NR_WRITE_QUEUES =1 <<17 ,NVMF_OPT_NR_POLL_QUEUES =1 <<18 ,NVMF_OPT_TOS =1 <<19 , + NVMF_OPT_FAIL_FAST_TMO = 1 << 20, };/**
@@ -89,6 +92,7 @@ enum {* @nr_write_queues: number of queues for write I/O * @nr_poll_queues: number of queues for polling I/O * @tos: type of service +* @fast_fail_tmo_ns: Fast I/O fail timeout in nanoseconds; */ struct nvmf_ctrl_options {unsigned mask;
@@ -111,6 +115,7 @@ struct nvmf_ctrl_options {unsigned int nr_write_queues;unsigned int nr_poll_queues;int tos; +unsigned int fail_fast_tmo; };/*
drivers/nvme/host/multipath.c +3 -1
@@ -281,9 +281,11 @@ static bool nvme_available_path(struct nvme_ns_head *head)list_for_each_entry_rcu (ns, &head->list, siblings) {switch (ns->ctrl->state) { +case NVME_CTRL_CONNECTING : +if (test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ns->ctrl->flags)) +break ;case NVME_CTRL_LIVE :case NVME_CTRL_RESETTING : -case NVME_CTRL_CONNECTING :/* fallthru */ return true;default :
drivers/nvme/host/nvme.h +3 -0
@@ -256,6 +256,7 @@ struct nvme_ctrl {struct work_struct scan_work;struct work_struct async_event_work;struct delayed_work ka_work; +struct delayed_work failfast_work;struct nvme_command ka_cmd;struct work_struct fw_act_work;unsigned long events;
@@ -289,6 +290,8 @@ struct nvme_ctrl {u16 icdoff;u16 maxcmd;int nr_reconnects; +unsigned long flags; +#define NVME_CTRL_FAILFAST_EXPIRED 0struct nvmf_ctrl_options *opts;struct page *discard_page;
-----------------
Regards,
Victor
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 6 lines · Victor Gladkov
Quoted 5 lines · Christoph Hellwig
Quoted 4 lines · Victor Gladkov
Quoted 2 lines · Victor Gladkov
1. Added multipath support for this patch.
2. Small refactoring (according to the review)
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
Anyone have any comments on the latest proposed patch?
Where is the latest patch? I didn't see a repost. To kick off a discussion please just resent it, in proper patch format and with the proper style to make reviewing it easier.
Patch updated for branch nvme/for-5.6
Christoph meant you should send a patch with a version bump in subject (e.g. [PATCH v3]) and a proper patch header. No maintainer can pick this patch up like you've posted (not without forcing them to do more work than they should).
Some nits below.
drivers/nvme/host/fabrics.h line 19
Quoted 10 lines
drivers/nvme/host/fabrics.h +2
-0@@ -15,6 +15,8 @@#define NVMF_DEF_RECONNECT_DELAY 10/* default to 600 seconds of reconnect attempts before giving up */ #define NVMF_DEF_CTRL_LOSS_TMO 600 +/* default to 30 seconds of fail fast IO commands */ +#define NVMF_DEF_FAIL_FAST_TMO 0
Your comment says "default to 30 seconds" yet in practice 0 is used?
Quoted 19 lines
drivers/nvme/host/fabrics.h +2
-0/* * Define a host as seen by the target. We allocate one at boot, but also @@ -56,6 +58,7 @@ enum {NVMF_OPT_NR_WRITE_QUEUES =1 <<17 ,NVMF_OPT_NR_POLL_QUEUES =1 <<18 ,NVMF_OPT_TOS =1 <<19 , + NVMF_OPT_FAIL_FAST_TMO = 1 << 20, };/** @@ -89,6 +92,7 @@ enum {* @nr_write_queues: number of queues for write I/O * @nr_poll_queues: number of queues for polling I/O * @tos: type of service +* @fast_fail_tmo_ns: Fast I/O fail timeout in nanoseconds; */ struct nvmf_ctrl_options {unsigned mask;
drivers/nvme/host/fabrics.h struct nvmf_ctrl_options { · line 119
Quoted 5 lines
drivers/nvme/host/fabrics.h +1
-0@@ -111,6 +115,7 @@ struct nvmf_ctrl_options {unsigned int nr_write_queues;unsigned int nr_poll_queues;int tos; +unsigned int fail_fast_tmo; };
Above you document "@fast_fail_tmo_ns" but then add the member with name "fail_fast_tmo".
drivers/nvme/host/multipath.c static bool nvme_available_path(struct nvme_ns_head *head) · line 286
Quoted 12 lines
drivers/nvme/host/fabrics.h +0
-0/* drivers/nvme/host/multipath.c +3
-0@@ -281,9 +281,11 @@ static bool nvme_available_path(struct nvme_ns_head *head)list_for_each_entry_rcu (ns, &head->list, siblings) {switch (ns->ctrl->state) { +case NVME_CTRL_CONNECTING : +if (test_bit (NVME_CTRL_FAILFAST_EXPIRED , &ns->ctrl->flags)) +break ;
James suggested this case order, when he did he included /* fallthru */, it helps to have it where appropriate (like this case).
Quoted 19 lines
drivers/nvme/host/multipath.c +0
-1case NVME_CTRL_LIVE :case NVME_CTRL_RESETTING : -case NVME_CTRL_CONNECTING :/* fallthru */ return true;default :drivers/nvme/host/nvme.h +1
-0@@ -256,6 +256,7 @@ struct nvme_ctrl {struct work_struct scan_work;struct work_struct async_event_work;struct delayed_work ka_work; +struct delayed_work failfast_work;struct nvme_command ka_cmd;struct work_struct fw_act_work;unsigned long events;
drivers/nvme/host/nvme.h struct nvme_ctrl { · line 297
Quoted 8 lines
drivers/nvme/host/nvme.h +2
-0@@ -289,6 +290,8 @@ struct nvme_ctrl {u16 icdoff;u16 maxcmd;int nr_reconnects; +unsigned long flags; +#define NVME_CTRL_FAILFAST_EXPIRED 0struct nvmf_ctrl_options *opts;struct page *discard_page;
Seems awkward to define bits for "flags" within the body of the struct; but I'll defer to others' judgement.
Mike
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme