Support for cancel command (TP4097a)
Subsystems (3)
v3 cover Support for cancel command (TP4097a)
This patchset adds support for the cancel commands
The nvme host driver could take advantage of Cancel (defined by TP 4097a) to abort a command that timed out instead of resetting the controller.
The abort command's implementation is mandated by the specs but the target isn't necessarily required to perform any meaningful action (as it happens, for example, with the Linux nvmet implementation, where the abort command does nothing at all).
On the other hand, the Cancel command is optional, therefore if the target claims to support it, it's reasonable to expect its implementation to perform a useful action.
This patchset modifies the tcp and rdma host drivers' timeout handlers to check if the target supports the Cancel command; if it does, it will use it to try to abort the command that timed out. If multiple commands time out at the same time, the driver will use a Cancel command with the 'multiple commands' flag to attempt to abort all stuck commands as a last attempt to avoid a controller reset.
If Cancel isn't supported, if the allocation of the command fails, or if any failure is encountered, the driver will fall back to the standard controller reset mechanism.
Example of dmesg output:
nvme nvme0: I/O tag 3 (3003) type 4 opcode 0x1 (I/O Cmd) QID 5 timeout nvme nvme0: I/O tag 4 (2004) type 4 opcode 0x1 (I/O Cmd) QID 5 timeout nvme nvme0: I/O tag 5 (4005) type 4 opcode 0x1 (I/O Cmd) QID 5 timeout nvme nvme0: Cancel status: 0x0 imm abrts = 1 def abrts = 0 nvme nvme0: I/O tag 6 (5006) type 4 opcode 0x1 (I/O Cmd) QID 5 timeout nvme nvme0: Cancel status: 0x0 imm abrts = 63 def abrts = 0 nvme nvme0: I/O tag 3 (9003) type 4 opcode 0x1 (I/O Cmd) QID 6 timeout nvme nvme0: I/O tag 4 (a004) type 4 opcode 0x1 (I/O Cmd) QID 6 timeout nvme nvme0: I/O tag 5 (a005) type 4 opcode 0x1 (I/O Cmd) QID 6 timeout nvme nvme0: Cancel status: 0x0 imm abrts = 1 def abrts = 0 nvme nvme0: I/O tag 73 (7049) type 4 opcode 0x1 (I/O Cmd) QID 6 timeout nvme nvme0: Cancel status: 0x0 imm abrts = 5 def abrts = 0
This also adds a target side Cancel implementation written by Chris Leech, intended for testing. It adds a new Kconfig option CONFIG_NVME_TARGET_DELAY_REQUESTS which depends on debugfs. When this config option is enabled, a delay attribute is added under the kernel/debug/nvmet/test-nqn/ctrl1/ directory.
When a controller is created, it will read as "0 0"
$ cat delay 0 0
The first number is a count of how many requests will be delayed. The second is a delay time in ms.
V3: In the host driver, instead of counting outstanding Cancel
commands using atomic_inc()/atomic_dec(), use two flags in the queue structure. (Suggested by Sagi Grimberg). Added a target side implementation.
V2: Reserve 2 commands per queue for cancel commands (the number is fixed but in future it can be made configurable with trivial changes to the code).
Chris Leech (5):
nvmet: put all nvmet_req.execute calls behind a function name
nvmet: add delay debugfs file to nvmet_ctrl
nvmet: delay requests
nvmet: command tracking
nvmet: cancel implementation
Maurizio Lombardi (6):
nvme: add definitions for cancel command
nvme-core: reserve tags for cancel commands
nvme-core: add a function to submit a cancel command
nvme-tcp: use the cancel command to perform an abort if target
supports it
nvme-rdma: use the cancel command to perform an abort if target
supports it
nvmet: emulate basic support for cancel commands
drivers/nvme/host/constants.c | 1 + drivers/nvme/host/core.c | 83 ++++++++++++++++++++++++++++- drivers/nvme/host/nvme.h | 4 ++ drivers/nvme/host/rdma.c | 50 +++++++++++++++-- drivers/nvme/host/tcp.c | 62 +++++++++++++++++++-- drivers/nvme/target/Kconfig | 9 ++++ drivers/nvme/target/Makefile | 1 + drivers/nvme/target/admin-cmd.c | 3 ++ drivers/nvme/target/core.c | 67 ++++++++++++++++++++++- drivers/nvme/target/debugfs.c | 40 ++++++++++++++ drivers/nvme/target/fc.c | 4 +- drivers/nvme/target/io-cmd-cancel.c | 77 ++++++++++++++++++++++++++ drivers/nvme/target/loop.c | 2 +- drivers/nvme/target/nvmet.h | 19 +++++++ drivers/nvme/target/rdma.c | 4 +- drivers/nvme/target/tcp.c | 4 +- include/linux/nvme.h | 22 ++++++++ 17 files changed, 434 insertions(+), 18 deletions(-) create mode 100644 drivers/nvme/target/io-cmd-cancel.c
Signature · 1 line
2.43.5
RFC PATCH v3 1/11 nvme: add definitions for cancel command
Add new definitions needed to support TP4097a.
--- drivers/nvme/host/constants.c | 1 + drivers/nvme/host/core.c | 1 + include/linux/nvme.h | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+)
drivers/nvme/host/constants.c +1 -0
@@ -19,6 +19,7 @@ static const char * const nvme_ops[] = { [nvme_cmd_resv_report] = "Reservation Report", [nvme_cmd_resv_acquire] = "Reservation Acquire", [nvme_cmd_resv_release] = "Reservation Release", + [nvme_cmd_cancel] = "Cancel", [nvme_cmd_zone_mgmt_send] = "Zone Management Send", [nvme_cmd_zone_mgmt_recv] = "Zone Management Receive", [nvme_cmd_zone_append] = "Zone Append",
drivers/nvme/host/core.c +1 -0
@@ -5087,6 +5087,7 @@ static inline void _nvme_check_size(void)BUILD_BUG_ON (sizeof (struct nvme_dsm_cmd ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_write_zeroes_cmd ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_abort_cmd ) !=64 ); +BUILD_BUG_ON (sizeof (struct nvme_cancel_cmd ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_get_log_page_command ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_command ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_id_ctrl ) !=NVME_IDENTIFY_DATA_SIZE );
include/linux/nvme.h +22 -0
@@ -28,6 +28,9 @@/* Special NSSR value, 'NVMe' */ #define NVME_SUBSYS_RESET 0x4E564D65 +/* Maximum number of reserved commands for Cancel */ +#define NVME_RSV_CANCEL_MAX 2 +enum nvme_subsys_type {/* Referral to another discovery type target subsystem */ NVME_NQN_DISC =1 ,
@@ -890,6 +893,7 @@ enum nvme_opcode { nvme_cmd_resv_report =0x0e , nvme_cmd_resv_acquire =0x11 , nvme_cmd_resv_release =0x15 , + nvme_cmd_cancel =0x18 , nvme_cmd_zone_mgmt_send =0x79 , nvme_cmd_zone_mgmt_recv =0x7a , nvme_cmd_zone_append = 0x7d,
@@ -1425,6 +1429,22 @@ struct nvme_abort_cmd {__u32 rsvd11[5 ]; }; +struct nvme_cancel_cmd { +__u8 opcode; +__u8 flags; +__u16 command_id; +__le32 nsid; +__u32 rsvd1[8 ]; +__le16 sqid; +__u16 cid; +__u8 action; +__u8 rsvd11[3 ]; +__u32 rsvd12[4 ]; +}; + +#define NVME_CANCEL_ACTION_MUL_CMD 1 +#define NVME_CANCEL_ACTION_SINGLE_CMD 0 +struct nvme_download_firmware {__u8 opcode;__u8 flags;
@@ -1887,6 +1907,7 @@ struct nvme_command {struct nvme_zone_mgmt_send_cmd zms;struct nvme_zone_mgmt_recv_cmd zmr;struct nvme_abort_cmd abort; +struct nvme_cancel_cmd cancel;struct nvme_get_log_page_command get_log_page;struct nvmf_common_command fabrics;struct nvmf_connect_command connect;
@@ -2069,6 +2090,7 @@ enum {NVME_SC_INVALID_PI =0x181 ,NVME_SC_READ_ONLY =0x182 ,NVME_SC_ONCS_NOT_SUPPORTED =0x183 , + NVME_SC_INVALID_CID = 0x184,/* * I/O Command Set Specific - Fabrics commands:
Signature · 1 line
2.43.5
RFC PATCH v3 2/11 nvme-core: reserve tags for cancel commands
In reply to Maurizio LombardiIf the controller supports cancel commands, reserve NVME_RSV_CANCEL_MAX tags for them.
--- drivers/nvme/host/core.c | 15 ++++++++++++++- drivers/nvme/host/nvme.h | 1 + 2 files changed, 15 insertions(+), 1 deletion(-)
drivers/nvme/host/core.c +14 -1
@@ -1210,6 +1210,14 @@ u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) }EXPORT_SYMBOL_NS_GPL (nvme_command_effects,"NVME_TARGET_PASSTHRU" ); +bool nvme_io_command_supported (struct nvme_ctrl *ctrl,u8 opcode) +{ +u32 effects =le32_to_cpu (ctrl->effects->iocs[opcode]); + +return effects &NVME_CMD_EFFECTS_CSUPP ; +} +EXPORT_SYMBOL_GPL (nvme_io_command_supported); +u32 nvme_passthru_start (struct nvme_ctrl *ctrl,struct nvme_ns *ns,u8 opcode) {u32 effects =nvme_command_effects (ctrl, ns, opcode);
@@ -4675,9 +4683,14 @@ int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,*/ if (ctrl->quirks &NVME_QUIRK_SHARED_TAGS ) set->reserved_tags =NVME_AQ_DEPTH ; -else if (ctrl->ops->flags &NVME_F_FABRICS ) +else if (ctrl->ops->flags &NVME_F_FABRICS ) {/* Reserved for fabric connect */ set->reserved_tags =1 ; +if (nvme_io_command_supported (ctrl, nvme_cmd_cancel)) { +/* Reserved for cancel commands */ + set->reserved_tags +=NVME_RSV_CANCEL_MAX ; + } + } set->numa_node = ctrl->numa_node;if (ctrl->ops->flags &NVME_F_BLOCKING ) set->flags |=BLK_MQ_F_BLOCKING ;
drivers/nvme/host/nvme.h +1 -0
@@ -1187,6 +1187,7 @@ static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {};u32 nvme_command_effects (struct nvme_ctrl *ctrl,struct nvme_ns *ns,u8 opcode); +bool nvme_io_command_supported (struct nvme_ctrl *ctrl,u8 opcode);u32 nvme_passthru_start (struct nvme_ctrl *ctrl,struct nvme_ns *ns,u8 opcode);int nvme_execute_rq (struct request *rq,bool at_head); void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
Signature · 1 line
2.43.5
RFC PATCH v3 3/11 nvme-core: add a function to submit a cancel command
In reply to Maurizio LombardiAdd a function to send a cancel command to abort the specified request. When the cancel command completes, the host driver will print the number of deferred and immediate aborts performed by the target.
--- drivers/nvme/host/core.c | 61 ++++++++++++++++++++++++++++++++++++++++ drivers/nvme/host/nvme.h | 2 ++ 2 files changed, 63 insertions(+)
drivers/nvme/host/core.c +61 -0
@@ -3113,6 +3113,67 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,return nvme_submit_sync_cmd (ctrl->admin_q, &c, log, size); } +static enum rq_end_io_ret nvme_cancel_endio (struct request *req,blk_status_t error) +{ +struct nvme_ctrl *ctrl = req->end_io_data; +u32 result; +u16 imm_abrts, def_abrts; + +if (!error) { + result =le32_to_cpu (nvme_req (req)->result.u32); + def_abrts =upper_16_bits (result); + imm_abrts =lower_16_bits (result); + +dev_warn (ctrl->device, +"Cancel status: 0x0 imm abrts = %u def abrts = %u" , + imm_abrts, def_abrts); + }else { +dev_warn (ctrl->device,"Cancel status: 0x%x" , +nvme_req (req)->status); + } + +blk_mq_free_request (req); +return RQ_END_IO_NONE ; +} + +int nvme_submit_cancel_req (struct nvme_ctrl *ctrl,struct request *rq, +unsigned int sqid,int action) +{ +struct nvme_command c = { }; +struct request *cancel_req; + +if (sqid ==0 ) +return -EINVAL ; + + c.cancel.opcode = nvme_cmd_cancel; + c.cancel.sqid =cpu_to_le32 (sqid); + c.cancel.nsid =NVME_NSID_ALL ; + c.cancel.action = action; +if (action ==NVME_CANCEL_ACTION_SINGLE_CMD ) + c.cancel.cid =nvme_cid (rq); +else + c.cancel.cid =0xFFFF ; + + cancel_req =blk_mq_alloc_request_hctx (rq->q,nvme_req_op (&c), +BLK_MQ_REQ_NOWAIT | +BLK_MQ_REQ_RESERVED , + sqid -1 ); +if (IS_ERR (cancel_req)) { +dev_warn (ctrl->device,"%s: Could not allocate the Cancel " +"command" , __func__); +return PTR_ERR (cancel_req); + } + +nvme_init_request (cancel_req, &c); + cancel_req->end_io = nvme_cancel_endio; + cancel_req->end_io_data = ctrl; + +blk_execute_rq_nowait (cancel_req, false); + +return 0 ; +} +EXPORT_SYMBOL_GPL (nvme_submit_cancel_req); +static int nvme_get_effects_log (struct nvme_ctrl *ctrl,u8 csi,struct nvme_effects_log **log) {
drivers/nvme/host/nvme.h +2 -0
@@ -908,6 +908,8 @@ int nvme_delete_ctrl(struct nvme_ctrl *ctrl);void nvme_queue_scan (struct nvme_ctrl *ctrl);int nvme_get_log (struct nvme_ctrl *ctrl,u32 nsid,u8 log_page,u8 lsp,u8 csi,void *log,size_t size,u64 offset); +int nvme_submit_cancel_req (struct nvme_ctrl *ctrl,struct request *rq, +unsigned int sqid,int action);bool nvme_tryget_ns_head (struct nvme_ns_head *head);void nvme_put_ns_head (struct nvme_ns_head *head); int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
Signature · 1 line
2.43.5
RFC PATCH v3 4/11 nvme-tcp: use the cancel command to perform an abort if target supports it
In reply to Maurizio LombardiIf available, use Cancel command to abort the command that timed out instead of resetting the controller.
Limit the number of outstanding cancel commands
to a maximum of 2 per queue;
If more than one command time out at the same time, the driver will use a Cancel command with action flag set to "Multiple commands" to abort all the commands on the specified queue as a last resort to avoid a controller reset.
If the cancel command is not supported or if any error is encountered, the driver will fall back to the normal controller reset.
--- drivers/nvme/host/core.c | 6 ++++ drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/tcp.c | 62 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 64 insertions(+), 5 deletions(-)
drivers/nvme/host/core.c +6 -0
@@ -1218,6 +1218,12 @@ bool nvme_io_command_supported(struct nvme_ctrl *ctrl, u8 opcode) }EXPORT_SYMBOL_GPL (nvme_io_command_supported); +bool nvme_is_cancel (struct nvme_command *cmd) +{ +return cmd->common.opcode == nvme_cmd_cancel; +} +EXPORT_SYMBOL_GPL (nvme_is_cancel); +u32 nvme_passthru_start (struct nvme_ctrl *ctrl,struct nvme_ns *ns,u8 opcode) {u32 effects =nvme_command_effects (ctrl, ns, opcode);
drivers/nvme/host/nvme.h +1 -0
@@ -1190,6 +1190,7 @@ static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {};u32 nvme_command_effects (struct nvme_ctrl *ctrl,struct nvme_ns *ns,u8 opcode);bool nvme_io_command_supported (struct nvme_ctrl *ctrl,u8 opcode); +bool nvme_is_cancel (struct nvme_command *cmd);u32 nvme_passthru_start (struct nvme_ctrl *ctrl,struct nvme_ns *ns,u8 opcode);int nvme_execute_rq (struct request *rq,bool at_head); void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
drivers/nvme/host/tcp.c +57 -5
131 lines not on this page · raw message
Signature · 1 line
2.43.5
RFC PATCH v3 5/11 nvme-rdma: use the cancel command to perform an abort if target supports it
In reply to Maurizio LombardiIf available, use Cancel command to abort the command that timed out instead of resetting the controller.
Limit the number of outstanding cancel commands
to a maximum of 2 per queue;
If more than one command time out at the same time, the driver will use a Cancel command with action flag set to "Multiple commands" to abort all the commands on the specified queue as a last resort to avoid a controller reset.
If the cancel command is not supported or if any error is encountered, the driver will fall back to the normal controller reset.
--- drivers/nvme/host/rdma.c | 50 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-)
drivers/nvme/host/rdma.c +46 -4
@@ -74,12 +74,15 @@ struct nvme_rdma_request {struct nvme_rdma_sgl data_sgl;struct nvme_rdma_sgl *metadata_sgl;bool use_sig_mr; +bool aborted; };enum nvme_rdma_queue_flags {NVME_RDMA_Q_ALLOCATED =0 ,NVME_RDMA_Q_LIVE =1 ,NVME_RDMA_Q_TR_READY =2 , +NVME_RDMA_Q_CANCEL_ONE =3 , +NVME_RDMA_Q_CANCEL_ALL =4 , };struct nvme_rdma_queue {
@@ -619,6 +622,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl, }set_bit (NVME_RDMA_Q_ALLOCATED , &queue->flags); +clear_bit (NVME_RDMA_Q_CANCEL_ONE , &queue->flags); +clear_bit (NVME_RDMA_Q_CANCEL_ALL , &queue->flags);return 0 ;
@@ -1954,16 +1959,18 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq) {struct nvme_rdma_request *req =blk_mq_rq_to_pdu (rq);struct nvme_rdma_queue *queue = req->queue; -struct nvme_rdma_ctrl *ctrl = queue->ctrl; +struct nvme_rdma_ctrl *rdma_ctrl = queue->ctrl; +struct nvme_ctrl *ctrl = &rdma_ctrl->ctrl;struct nvme_command *cmd = req->req.cmd;int qid =nvme_rdma_queue_idx (queue); +int error, action; -dev_warn (ctrl->ctrl.device, +dev_warn (ctrl->device,"I/O tag %d (%04x) opcode %#x (%s) QID %d timeout\n" , rq->tag,nvme_cid (rq), cmd->common.opcode,nvme_fabrics_opcode_str (qid, cmd), qid); -if (nvme_ctrl_state (&ctrl->ctrl) !=NVME_CTRL_LIVE ) { +if (nvme_ctrl_state (ctrl) !=NVME_CTRL_LIVE ) {/* * If we are resetting, connecting or deleting we should * complete immediately because we may block controller
@@ -1981,11 +1988,40 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)return BLK_EH_DONE ; } +if (!req->aborted) { +if (!nvme_io_command_supported (ctrl, nvme_cmd_cancel) || !qid) + goto err_recovery; + +if (!test_and_set_bit (NVME_RDMA_Q_CANCEL_ONE , &queue->flags)) { + action =NVME_CANCEL_ACTION_SINGLE_CMD ; + }else if (!test_and_set_bit (NVME_RDMA_Q_CANCEL_ALL , + &queue->flags)) { + action =NVME_CANCEL_ACTION_MUL_CMD ; + }else { +/* No free reserved commands. +* this means a "multiple commands" cancel +* is currently under execution and this request +* is likely to be canceled. Mark this +* request as aborted and reset the timer. +*/ + goto abort; + } + + error =nvme_submit_cancel_req (ctrl, rq, qid, action); +if (error) + goto err_recovery; + +abort: + req->aborted = true; +return BLK_EH_RESET_TIMER ; + } +/* * LIVE state should trigger the normal error recovery which will * handle completing this request. */ -nvme_rdma_error_recovery (ctrl); +err_recovery: +nvme_rdma_error_recovery (rdma_ctrl);return BLK_EH_RESET_TIMER ; }
@@ -2009,6 +2045,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,return nvme_fail_nonready_command (&queue->ctrl->ctrl, rq); dev = queue->device->dev; + req->aborted = false; req->sqe.dma = ib_dma_map_single(dev, req->sqe.data, sizeof(struct nvme_command),
@@ -2113,6 +2150,7 @@ static void nvme_rdma_complete_rq(struct request *rq)struct nvme_rdma_request *req =blk_mq_rq_to_pdu (rq);struct nvme_rdma_queue *queue = req->queue;struct ib_device *ibdev = queue->device->dev; +bool is_cancel =nvme_is_cancel (req->req.cmd);if (req->use_sig_mr)nvme_rdma_check_pi_status (req);
@@ -2121,6 +2159,10 @@ static void nvme_rdma_complete_rq(struct request *rq)ib_dma_unmap_single (ibdev, req->sqe.dma,sizeof (struct nvme_command ),DMA_TO_DEVICE );nvme_complete_rq (rq); +if (is_cancel) { +if (!test_and_clear_bit (NVME_RDMA_Q_CANCEL_ALL , &queue->flags)) +clear_bit (NVME_RDMA_Q_CANCEL_ONE , &queue->flags); + } }static void nvme_rdma_map_queues (struct blk_mq_tag_set *set)
Signature · 1 line
2.43.5
RFC PATCH v3 6/11 nvmet: emulate basic support for cancel commands
In reply to Maurizio LombardiAdd a module parameter that allows the user to enable a basic cancel command emulation, this could be useful for testing the host driver's cancel command implementation.
It just reports that no abort has been executed, it will however do some basic sanity check on the command sent by the host and return an error if the fields are invalid.
--- drivers/nvme/target/Makefile | 2 +- drivers/nvme/target/admin-cmd.c | 4 +++ drivers/nvme/target/core.c | 11 ++++++ drivers/nvme/target/io-cmd-cancel.c | 52 +++++++++++++++++++++++++++++ drivers/nvme/target/nvmet.h | 4 +++ 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 drivers/nvme/target/io-cmd-cancel.c
drivers/nvme/target/Makefile +1 -1
@@ -11,7 +11,7 @@ obj-$(CONFIG_NVME_TARGET_TCP) += nvmet-tcp.o obj-$(CONFIG_NVME_TARGET_PCI_EPF) += nvmet-pci-epf.o nvmet-y += core.o configfs.o admin-cmd.o fabrics-cmd.o \ - discovery.o io-cmd-file.o io-cmd-bdev.o pr.o + discovery.o io-cmd-file.o io-cmd-bdev.o pr.o io-cmd-cancel.o nvmet-$(CONFIG_NVME_TARGET_DEBUGFS) += debugfs.o nvmet-$(CONFIG_NVME_TARGET_PASSTHRU) += passthru.o nvmet-$(CONFIG_BLK_DEV_ZONED) += zns.o
drivers/nvme/target/admin-cmd.c +4 -0
@@ -398,6 +398,10 @@ static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log) log->iocs[nvme_cmd_resv_release] = log->iocs[nvme_cmd_resv_report] =cpu_to_le32 (NVME_CMD_EFFECTS_CSUPP ); +if (emulate_cancel_support) { + log->iocs[nvme_cmd_cancel] = +cpu_to_le32 (NVME_CMD_EFFECTS_CSUPP ); + } log->iocs[nvme_cmd_write] = log->iocs[nvme_cmd_write_zeroes] =cpu_to_le32 (NVME_CMD_EFFECTS_CSUPP |NVME_CMD_EFFECTS_LBCC );
drivers/nvme/target/core.c +11 -0
@@ -27,6 +27,10 @@ static DEFINE_IDA(cntlid_ida);struct workqueue_struct *nvmet_wq;EXPORT_SYMBOL_GPL (nvmet_wq); +bool emulate_cancel_support; +module_param (emulate_cancel_support, bool,0644 ); +MODULE_PARM_DESC (emulate_cancel_support,"Emulate the cancel command support. Default = false" ); +/* * This read/write semaphore is used to synchronize access to configuration * information on a target system that will result in discovery log page
@@ -1057,6 +1061,13 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)if (nvmet_is_passthru_req (req))return nvmet_parse_passthru_io_cmd (req); +if (emulate_cancel_support && + req->cmd->common.opcode == nvme_cmd_cancel) { + req->execute = nvmet_execute_cancel; +if (req->cmd->cancel.nsid ==NVME_NSID_ALL ) +return 0 ; + } + ret =nvmet_req_find_ns (req);if (unlikely (ret))return ret;
drivers/nvme/target/io-cmd-cancel.c new file +52 -0
52 lines not on this page · raw message
drivers/nvme/target/nvmet.h +4 -0
@@ -40,6 +40,8 @@#define nvmet_for_each_enabled_ns (xa, index, entry) \ xa_for_each_marked(xa, index, entry, NVMET_NS_ENABLED) +extern bool emulate_cancel_support; +/* * Supported optional AENs: */
@@ -714,6 +716,8 @@ void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req);void nvmet_bdev_execute_zone_mgmt_send (struct nvmet_req *req);void nvmet_bdev_execute_zone_append (struct nvmet_req *req); +void nvmet_execute_cancel (struct nvmet_req *req); +static inline u32 nvmet_rw_data_len (struct nvmet_req *req) { return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) <<
Signature · 1 line
2.43.5
RFC PATCH v3 7/11 nvmet: put all nvmet_req.execute calls behind a function name
In reply to Maurizio LombardiFrom: Chris Leech <[email protected]>
This is setup for being able to manipluate the outstanding requests.
--- drivers/nvme/target/fc.c | 4 ++-- drivers/nvme/target/loop.c | 2 +- drivers/nvme/target/nvmet.h | 1 + drivers/nvme/target/rdma.c | 4 ++-- drivers/nvme/target/tcp.c | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-)
drivers/nvme/target/fc.c +2 -2
@@ -2378,7 +2378,7 @@ nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod) }/* data transfer complete, resume with nvmet layer */ - fod->req.execute (&fod->req); +nvmet_execute_request (&fod->req);break ;case NVMET_FCOP_READDATA :
@@ -2589,7 +2589,7 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,* can invoke the nvmet_layer now. If read data, cmd completion will * push the data */ - fod->req.execute (&fod->req); +nvmet_execute_request (&fod->req);return ; transport_error:
drivers/nvme/target/loop.c +1 -1
@@ -125,7 +125,7 @@ static void nvme_loop_execute_work(struct work_struct *work)struct nvme_loop_iod *iod = container_of(work, struct nvme_loop_iod, work); - iod->req.execute (&iod->req); +nvmet_execute_request (&iod->req); } static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
drivers/nvme/target/nvmet.h +1 -0
@@ -967,4 +967,5 @@ struct nvmet_feat_arbitration { u8 ab; }; +static inline void nvmet_execute_request (struct nvmet_req *req) { req->execute (req); }#endif /* _NVMET_H */
drivers/nvme/target/rdma.c +2 -2
@@ -773,7 +773,7 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc)if (unlikely (status))nvmet_req_complete (&rsp->req, status);else - rsp->req.execute (&rsp->req); +nvmet_execute_request (&rsp->req); }static void nvmet_rdma_write_data_done (struct ib_cq *cq,struct ib_wc *wc)
@@ -958,7 +958,7 @@ static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp) queue->cm_id->port_num, &rsp->read_cqe, NULL))nvmet_req_complete (&rsp->req,NVME_SC_DATA_XFER_ERROR ); }else { - rsp->req.execute (&rsp->req); +nvmet_execute_request (&rsp->req); }return true;
drivers/nvme/target/tcp.c +2 -2
@@ -597,7 +597,7 @@ static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd)if (unlikely (cmd->flags &NVMET_TCP_F_INIT_FAILED ))nvmet_tcp_queue_response (&cmd->req);else - cmd->req.execute (&cmd->req); +nvmet_execute_request (&cmd->req); }static int nvmet_try_send_data_pdu (struct nvmet_tcp_cmd *cmd)
@@ -1104,7 +1104,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) goto out; } - queue->cmd->req.execute (&queue->cmd->req); +nvmet_execute_request (&queue->cmd->req); out:nvmet_prepare_receive_pdu (queue);return ret;
Signature · 1 line
2.43.5
RFC PATCH v3 8/11 nvmet: add delay debugfs file to nvmet_ctrl
In reply to Maurizio LombardiFrom: Chris Leech <[email protected]>
Creates an delay attribute file on the controler in debugfs /sys/kernel/debug/nvmet/<subsystem>/ctrlN/delay
Reading this file returns two numbers, a reqeust delay count and a delay time in ms. Each delayed request will decrement the delay count until it reaches 0.
Writing to this file can set both the delay and count at once, or just the count to trigger more delays.
# delay the next 5 request by 5 seconds each echo 5 5000 > delay
# set the delay time to 3 seconds without starting a count echo 0 3000 > delay
# delay to the next 5 requests by the current delay time echo 5 > delay
--- drivers/nvme/target/Kconfig | 9 ++++++++ drivers/nvme/target/debugfs.c | 40 +++++++++++++++++++++++++++++++++++ drivers/nvme/target/nvmet.h | 4 ++++ 3 files changed, 53 insertions(+)
drivers/nvme/target/Kconfig +9 -0
@@ -126,3 +126,12 @@ config NVME_TARGET_PCI_EPF capable PCI controller. If unsure, say N. + +config NVME_TARGET_DELAY_REQUESTS + bool "NVMe over Fabrics target request delay" + depends on NVME_TARGET && NVME_TARGET_DEBUGFS + help + This is a testing feature to allow delaying request completion in an + NVMe over Fabrics target, which allows for support of the cancel command. + + If unsure, say N.
drivers/nvme/target/debugfs.c +40 -0
@@ -153,6 +153,42 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p)NVMET_DEBUGFS_ATTR (nvmet_ctrl_tls_concat);#endif +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +static int nvmet_ctrl_delay_show (struct seq_file *m,void *p) +{ +struct nvmet_ctrl *ctrl = m->private; +int delay_count =atomic_read (&ctrl->delay_count); + +seq_printf (m,"%u %u\n" , delay_count, ctrl->delay_msec); +return 0 ; +} + +static ssize_t nvmet_ctrl_delay_write (struct file *file,const char __user *buf, +size_t count,loff_t *ppos) +{ +struct seq_file *m = file->private_data; +struct nvmet_ctrl *ctrl = m->private; +char delay_buf[22 ] = {}; +int delay_count; +int delay_msec; +int n; + +if (count >=sizeof (delay_buf)) +return -EINVAL ; +if (copy_from_user (delay_buf, buf, count)) +return -EFAULT ; + + n =sscanf (delay_buf,"%u %u" , &delay_count, &delay_msec); +if (n <1 || n >2 ) +return -EINVAL ; +if (n ==2 ) + ctrl->delay_msec = delay_msec; +atomic_set (&ctrl->delay_count, delay_count); +return count; +} +NVMET_DEBUGFS_RW_ATTR (nvmet_ctrl_delay); +#endif /* CONFIG_NVME_TARGET_DELAY_REQUESTS */ +int nvmet_debugfs_ctrl_setup (struct nvmet_ctrl *ctrl) {char name[32 ];
@@ -183,6 +219,10 @@ int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) &nvmet_ctrl_tls_concat_fops);debugfs_create_file ("tls_key" ,S_IRUSR , ctrl->debugfs_dir, ctrl, &nvmet_ctrl_tls_key_fops); +#endif +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +debugfs_create_file ("delay" ,S_IWUSR , ctrl->debugfs_dir, ctrl, + &nvmet_ctrl_delay_fops);#endif return 0 ; }
drivers/nvme/target/nvmet.h +4 -0
@@ -306,6 +306,10 @@ struct nvmet_ctrl {#endif #ifdef CONFIG_NVME_TARGET_TCP_TLS struct key *tls_key; +#endif +#ifdef CONFIG_NVME_TARGET_DELAY_REQUESTS +atomic_t delay_count; +u32 delay_msec;#endif struct nvmet_pr_log_mgr pr_log_mgr; };
Signature · 1 line
2.43.5
RFC PATCH v3 9/11 nvmet: delay requests
In reply to Maurizio LombardiFrom: Chris Leech <[email protected]>
Allow requests to be delayed for testing.
--- drivers/nvme/target/core.c | 30 ++++++++++++++++++++++++++++++ drivers/nvme/target/nvmet.h | 8 ++++++++ 2 files changed, 38 insertions(+)
drivers/nvme/target/core.c +30 -0
@@ -6,7 +6,9 @@#define pr_fmt (fmt) KBUILD_MODNAME ": " fmt#include <linux/module.h> #include <linux/random.h> +#include <linux/jiffies.h> #include <linux/rculist.h> +#include <linux/workqueue.h> #include <linux/pci-p2pdma.h> #include <linux/scatterlist.h>
@@ -1753,6 +1755,34 @@ ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl,return ctrl->ops->host_traddr (ctrl, traddr, traddr_len); } +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +static void nvmet_delayed_execute_req (struct work_struct *work) { +struct nvmet_req *req = + container_of(to_delayed_work(work), struct nvmet_req, req_work); + req->execute (req); +} + +void nvmet_execute_request (struct nvmet_req *req) { +struct nvmet_ctrl *ctrl = req->sq->ctrl; +int delay_count; +u32 delay_msec; + +if (unlikely (req->sq->qid ==0 )) +return req->execute (req); + +if (ctrl) { + delay_count =atomic_dec_if_positive (&ctrl->delay_count) +1 ; + delay_msec = ctrl->delay_msec; + } +if (!(ctrl && delay_count && delay_msec)) +return req->execute (req); + +INIT_DELAYED_WORK (&req->req_work, nvmet_delayed_execute_req); +queue_delayed_work (nvmet_wq, &req->req_work,msecs_to_jiffies (delay_msec)); +} +EXPORT_SYMBOL_GPL (nvmet_execute_request); +#endif +static struct nvmet_subsys *nvmet_find_get_subsys (struct nvmet_port *port,const char *subsysnqn) {
drivers/nvme/target/nvmet.h +8 -0
@@ -496,6 +496,9 @@ struct nvmet_req {u16 error_loc;u64 error_slba;struct nvmet_pr_per_ctrl_ref *pc_ref; +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +struct delayed_work req_work; +#endif };#define NVMET_MAX_MPOOL_BVEC 16
@@ -971,5 +974,10 @@ struct nvmet_feat_arbitration { u8 ab; }; +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +void nvmet_execute_request (struct nvmet_req *req); +#else static inline void nvmet_execute_request (struct nvmet_req *req) { req->execute (req); } +#endif +#endif /* _NVMET_H */
Signature · 1 line
2.43.5
RFC PATCH v3 10/11 nvmet: command tracking
In reply to Maurizio LombardiFrom: Chris Leech <[email protected]>
Enable tracking of all outstanding requests in an XArray
--- drivers/nvme/target/core.c | 29 ++++++++++++++++++++++++++++- drivers/nvme/target/nvmet.h | 4 ++++ 2 files changed, 32 insertions(+), 1 deletion(-)
drivers/nvme/target/core.c +28 -1
@@ -809,10 +809,23 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)nvmet_put_namespace (ns); } +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +static void nvmet_delayed_execute_req (struct work_struct *work); +#endif +void nvmet_req_complete (struct nvmet_req *req,u16 status) {struct nvmet_sq *sq = req->sq; +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +unsigned long flags; +/* only need to update the xarray if this was a delayed request */ +if (req->req_work.work.func == nvmet_delayed_execute_req) { +xa_lock_irqsave (&sq->outstanding_requests, flags); +__xa_erase (&sq->outstanding_requests, req->cmd->common.command_id); +xa_unlock_irqrestore (&sq->outstanding_requests, flags); + } +#endif __nvmet_req_complete (req, status);percpu_ref_put (&sq->ref); }
@@ -984,7 +997,9 @@ int nvmet_sq_init(struct nvmet_sq *sq)init_completion (&sq->free_done);init_completion (&sq->confirm_done);nvmet_auth_sq_init (sq); - +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +xa_init_flags (&sq->outstanding_requests,XA_FLAGS_LOCK_IRQ ); +#endif return 0 ; }EXPORT_SYMBOL_GPL (nvmet_sq_init);
@@ -1766,6 +1781,7 @@ void nvmet_execute_request(struct nvmet_req *req) {struct nvmet_ctrl *ctrl = req->sq->ctrl;int delay_count;u32 delay_msec; +unsigned long flags;if (unlikely (req->sq->qid ==0 ))return req->execute (req);
@@ -1777,6 +1793,17 @@ void nvmet_execute_request(struct nvmet_req *req) {if (!(ctrl && delay_count && delay_msec))return req->execute (req); +xa_lock_irqsave (&req->sq->outstanding_requests, flags); +int ret =__xa_insert (&req->sq->outstanding_requests, + req->cmd->common.command_id, req,GFP_KERNEL ); +xa_unlock_irqrestore (&req->sq->outstanding_requests, flags); + +if (ret) { +pr_err ("nvmet: failure to delay command %d" , + req->cmd->common.command_id); +return req->execute (req); + } +INIT_DELAYED_WORK (&req->req_work, nvmet_delayed_execute_req);queue_delayed_work (nvmet_wq, &req->req_work,msecs_to_jiffies (delay_msec)); }
drivers/nvme/target/nvmet.h +4 -0
@@ -172,6 +172,9 @@ struct nvmet_sq {#endif struct completion free_done;struct completion confirm_done; +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +struct xarray outstanding_requests; +#endif };struct nvmet_ana_group {
@@ -573,6 +576,7 @@ size_t nvmet_req_transfer_len(struct nvmet_req *req);bool nvmet_check_transfer_len (struct nvmet_req *req,size_t len);bool nvmet_check_data_len_lte (struct nvmet_req *req,size_t data_len);void nvmet_req_complete (struct nvmet_req *req,u16 status); +void nvmet_req_complete_delayed (struct nvmet_req *req,u16 status);int nvmet_req_alloc_sgls (struct nvmet_req *req);void nvmet_req_free_sgls (struct nvmet_req *req);
Signature · 1 line
2.43.5
RFC PATCH v3 11/11 nvmet: cancel implementation
In reply to Maurizio LombardiFrom: Chris Leech <[email protected]>
Requests are canceled if the have been delayed and cancel_delayed_work is successfull.
replace emulate_cancel_commands modparam with the delay kconfig
--- drivers/nvme/target/Makefile | 3 ++- drivers/nvme/target/admin-cmd.c | 7 +++---- drivers/nvme/target/core.c | 9 +++------ drivers/nvme/target/io-cmd-cancel.c | 27 ++++++++++++++++++++++++++- drivers/nvme/target/nvmet.h | 2 -- 5 files changed, 34 insertions(+), 14 deletions(-)
drivers/nvme/target/Makefile +2 -1
@@ -11,7 +11,8 @@ obj-$(CONFIG_NVME_TARGET_TCP) += nvmet-tcp.o obj-$(CONFIG_NVME_TARGET_PCI_EPF) += nvmet-pci-epf.o nvmet-y += core.o configfs.o admin-cmd.o fabrics-cmd.o \ - discovery.o io-cmd-file.o io-cmd-bdev.o pr.o io-cmd-cancel.o + discovery.o io-cmd-file.o io-cmd-bdev.o pr.o +nvmet-$(CONFIG_NVME_TARGET_DELAY_REQUESTS) += io-cmd-cancel.o nvmet-$(CONFIG_NVME_TARGET_DEBUGFS) += debugfs.o nvmet-$(CONFIG_NVME_TARGET_PASSTHRU) += passthru.o nvmet-$(CONFIG_BLK_DEV_ZONED) += zns.o
drivers/nvme/target/admin-cmd.c +3 -4
@@ -398,10 +398,9 @@ static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log) log->iocs[nvme_cmd_resv_release] = log->iocs[nvme_cmd_resv_report] =cpu_to_le32 (NVME_CMD_EFFECTS_CSUPP ); -if (emulate_cancel_support) { - log->iocs[nvme_cmd_cancel] = -cpu_to_le32 (NVME_CMD_EFFECTS_CSUPP ); - } +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) + log->iocs[nvme_cmd_cancel] =cpu_to_le32 (NVME_CMD_EFFECTS_CSUPP ); +#endif log->iocs[nvme_cmd_write] = log->iocs[nvme_cmd_write_zeroes] =cpu_to_le32 (NVME_CMD_EFFECTS_CSUPP |NVME_CMD_EFFECTS_LBCC );
drivers/nvme/target/core.c +3 -6
@@ -29,10 +29,6 @@ static DEFINE_IDA(cntlid_ida);struct workqueue_struct *nvmet_wq;EXPORT_SYMBOL_GPL (nvmet_wq); -bool emulate_cancel_support; -module_param (emulate_cancel_support, bool,0644 ); -MODULE_PARM_DESC (emulate_cancel_support,"Emulate the cancel command support. Default = false" ); -/* * This read/write semaphore is used to synchronize access to configuration * information on a target system that will result in discovery log page
@@ -1078,12 +1074,13 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)if (nvmet_is_passthru_req (req))return nvmet_parse_passthru_io_cmd (req); -if (emulate_cancel_support && - req->cmd->common.opcode == nvme_cmd_cancel) { +#if IS_ENABLED (CONFIG_NVME_TARGET_DELAY_REQUESTS ) +if (req->cmd->common.opcode == nvme_cmd_cancel) { req->execute = nvmet_execute_cancel;if (req->cmd->cancel.nsid ==NVME_NSID_ALL )return 0 ; } +#endif ret =nvmet_req_find_ns (req);if (unlikely (ret))
drivers/nvme/target/io-cmd-cancel.c +26 -1
@@ -14,6 +14,8 @@ void nvmet_execute_cancel(struct nvmet_req *req)int ret =0 ;struct nvmet_ctrl *ctrl = req->sq->ctrl;struct nvmet_sq *sq = req->sq; +struct nvmet_req *treq; +u32 canceled =0 ;if (!nvmet_check_transfer_len (req,0 ))return ;
@@ -45,8 +47,31 @@ void nvmet_execute_cancel(struct nvmet_req *req) ret =NVME_SC_INVALID_FIELD |NVME_STATUS_DNR ; } +if (!mult_cmds) { + treq =xa_load (&sq->outstanding_requests, cid); +if (treq) { +if (cancel_delayed_work (&treq->req_work)) { +pr_info ("nvmet: CANCEL success: %d" , cid); +nvmet_req_complete (treq,NVME_SC_ABORT_REQ ); + canceled +=1 ; + }else { +pr_info ("nvmet: CANCEL failed: %d" , cid); + } + }else { +pr_info ("nvmet: CANCEL request not found: %d" , cid); + } + }else { +unsigned long ucid; +xa_for_each (&sq->outstanding_requests, ucid, treq) { +if (cancel_delayed_work (&treq->req_work)) { +nvmet_req_complete (treq,NVME_SC_ABORT_REQ ); + canceled +=1 ; + } + } +pr_info ("nvmet: CANCEL removed %d requests" , canceled); + } exit: -nvmet_set_result (req,0 ); +nvmet_set_result (req, canceled);nvmet_req_complete (req, ret); }
drivers/nvme/target/nvmet.h +0 -2
@@ -40,8 +40,6 @@#define nvmet_for_each_enabled_ns (xa, index, entry) \ xa_for_each_marked(xa, index, entry, NVMET_NS_ENABLED) -extern bool emulate_cancel_support; -/* * Supported optional AENs: */
Signature · 1 line
2.43.5