Skip to contentSkip to conversation

nvme: call nvme_complete_rq on NVME_MPATH failures

PATCH Superseded James Smart 4 participants 9 messages Sep 21, 2018 – Sep 25, 2018 NVMe

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

When an io is rejected by nvmf_check_ready() due to validation of the controller state, the nvmf_fail_nonready_command() will normally return BLK_STS_RESOURCE to requeue and retry. However, if the controller is dying or the io is marked for NVME multipath, the io is failed so that the controller can terminate or so that the io can be issued on a different path. Unfortunately, as this reject point is before the transport has accepted the command, blk-mq ends up completing the io and never calls nvme_complete_rq(), which is where multipath may preserve/re-route the io. The end result is, the device user ends up seeing an EIO error.

Example: single path connectivity, controller is under load, and a reset is induced. An i/o is received: a) while the reset state has been set but the queues have yet to be stopped; or b) after queues are started (at end of reset) but before the reconnect has completed. The io finishes with an EIO status.

This patch modifies the reject point such that it appears to queue successfully, but actually completes the io with an error and calls nvme_complete_rq().

Signed-off-by James Smart <jsmart2021 at gmail.com>
---
 drivers/nvme/host/fabrics.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
drivers/nvme/host/fabrics.c +10 -1
index 206d63cb1afc..ed4b8e8e0249 100644
@@ -548,11 +548,20 @@ static struct nvmf_transport_ops *nvmf_lookup_transport(
 blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl,
 		struct request *rq)
 {
+	struct nvme_request *nvmerq = nvme_req(rq);
+
 	if (ctrl->state != NVME_CTRL_DELETING &&
 	    ctrl->state != NVME_CTRL_DEAD &&
 	    !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
 		return BLK_STS_RESOURCE;
-	nvme_req(rq)->status = NVME_SC_ABORT_REQ;
+
+	nvmerq->status = NVME_SC_ABORT_REQ;
+	nvmerq->result.u64 = 0;
+	if (rq->cmd_flags & REQ_NVME_MPATH) {
+		blk_mq_start_request(rq);
+		nvme_complete_rq(rq);
+		return BLK_STS_OK;
+	}
 	return BLK_STS_IOERR;
 }
 EXPORT_SYMBOL_GPL(nvmf_fail_nonready_command);
Signature · 1 line
2.13.1
maintainer
#1 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

drivers/nvme/host/fabrics.c static struct nvmf_transport_ops *nvmf_lookup_transport( · line 552

Quoted 11 lines
drivers/nvme/host/fabrics.c +2 -0
index 206d63cb1afc..ed4b8e8e0249 100644
@@ -548,11 +548,20 @@ static struct nvmf_transport_ops *nvmf_lookup_transport(
  blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl,
  		struct request *rq)
  {
+	struct nvme_request *nvmerq = nvme_req(rq);
+

Maybe simply call it req?

drivers/nvme/host/fabrics.c static struct nvmf_transport_ops *nvmf_lookup_transport( · line 558

Quoted 7 lines
drivers/nvme/host/fabrics.c +2 -1
  	if (ctrl->state != NVME_CTRL_DELETING &&
  	    ctrl->state != NVME_CTRL_DEAD &&
  	    !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
  		return BLK_STS_RESOURCE;
-	nvme_req(rq)->status = NVME_SC_ABORT_REQ;
+
+	nvmerq->status = NVME_SC_ABORT_REQ;

Not introduced in this patch, but should be le16_to_cpu() >> 1, but I guess the problem is that we need to open-code status setting here...

drivers/nvme/host/fabrics.c static struct nvmf_transport_ops *nvmf_lookup_transport( · line 562

Quoted 4 lines
drivers/nvme/host/fabrics.c +4 -0
+	nvmerq->result.u64 = 0;
+	if (rq->cmd_flags & REQ_NVME_MPATH) {
+		blk_mq_start_request(rq);
+		nvme_complete_rq(rq);

Why not calling nvme_end_request() directly instead?

You can also introduce marker helper for nvme_end_request to use:
static inline void nvme_mark_request(struct request *req, __le16 status,
union nvme_result result)

{
         struct nvme_request *rq = nvme_req(req);

         rq->status = le16_to_cpu(status) >> 1;
         rq->result = result;
}

Something like:

	if (rq->cmd_flags & REQ_NVME_MPATH) {
		blk_mq_start_request(rq);
		nvme_end_request(rq, NVME_SC_ABORT_REQ, res);
		return BLK_STS_OK;
	}

	nvme_mark_request(rq, NVME_SC_ABORT_REQ, res);
	return BLK_STS_IOERR;

Thoughts?

#2 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

Quoted 27 lines · Sagi Grimberg

drivers/nvme/host/fabrics.c static struct nvmf_transport_ops · line 553

Quoted 12 lines
drivers/nvme/host/fabrics.c +2 -0
index 206d63cb1afc..ed4b8e8e0249 100644
@@ -548,11 +548,20 @@ static struct nvmf_transport_ops 
 *nvmf_lookup_transport(
 ? blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl,
 ????????? struct request *rq)
 ? {
+??? struct nvme_request *nvmerq = nvme_req(rq);
+

Maybe simply call it req?

Quoted 7 lines

????? if (ctrl->state != NVME_CTRL_DELETING &&
????????? ctrl->state != NVME_CTRL_DEAD &&
????????? !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH)) ????????? return BLK_STS_RESOURCE;
-??? nvme_req(rq)->status = NVME_SC_ABORT_REQ;
+
+??? nvmerq->status = NVME_SC_ABORT_REQ;

Not introduced in this patch, but should be le16_to_cpu() >> 1, but I guess the problem is that we need to open-code status setting here...

Quoted 7 lines
Quoted 4 lines

+??? nvmerq->result.u64 = 0;
+??? if (rq->cmd_flags & REQ_NVME_MPATH) {
+??????? blk_mq_start_request(rq);
+??????? nvme_complete_rq(rq);

Why not calling nvme_end_request() directly instead?

Because then we have to do this curious twiddling with the 'status' field; when synthesizing the 'status' field we don't really have to set the 'phase' tag, only to have it stripped out again.

Quoted 19 lines

You can also introduce marker helper for nvme_end_request to use:
static inline void nvme_mark_request(struct request *req, __le16 status,
??????????????? union nvme_result result)
{
??????? struct nvme_request *rq = nvme_req(req);

??????? rq->status = le16_to_cpu(status) >> 1;
??????? rq->result = result;
}

Something like:

????if (rq->cmd_flags & REQ_NVME_MPATH) { ??????? blk_mq_start_request(rq);
??????? nvme_end_request(rq, NVME_SC_ABORT_REQ, res); ??????? return BLK_STS_OK;
????}

Quoted 5 lines

????nvme_mark_request(rq, NVME_SC_ABORT_REQ, res); ????return BLK_STS_IOERR;

Thoughts?

See above. I'd rather set the 'status' field directly.

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)
#3 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

In reply to Sagi Grimberg
Quoted 5 lines · Sagi Grimberg
Quoted 3 lines

+??? struct nvme_request *nvmerq = nvme_req(rq);
+

Maybe simply call it req?

Doesn't really matter. req vs rq, and having both, was too close for me. But not an issue if you want it something else.

Quoted 13 lines
Quoted 8 lines

????? if (ctrl->state != NVME_CTRL_DELETING &&
????????? ctrl->state != NVME_CTRL_DEAD &&
????????? !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH)) ????????? return BLK_STS_RESOURCE;
-??? nvme_req(rq)->status = NVME_SC_ABORT_REQ;
+
+??? nvmerq->status = NVME_SC_ABORT_REQ;

Not introduced in this patch, but should be le16_to_cpu() >> 1, but I guess the problem is that we need to open-code status setting here...

Quoted 6 lines
Quoted 4 lines

+??? nvmerq->result.u64 = 0;
+??? if (rq->cmd_flags & REQ_NVME_MPATH) {
+??????? blk_mq_start_request(rq);
+??????? nvme_complete_rq(rq);

Why not calling nvme_end_request() directly instead?

Well, I did both of these in the 1st iteration of the patch which I didn't post.? The issue was that nvme_end_request() ends up calling blk_mq_complete_request(), which calls the transport complete() routine. And the transport complete() routine is doing things assuming it went through its queue_rq() routine when, in this case, that never happened. For example, FC does some sglist cleanup and some put references, but the gets hadn't been done. I thought it better to completely ignore the transport all together since the transport never did anything on the request.? Thus, I collapsed the logic in nvme_end_request() with nvme_complete_rq().??? (note: normally transport does status<<1 while nvme_end_request does status >> 1 before calling nvme_complete_rq).

When following the calling sequences with blk-mq, I was really surprised that the blk-mq layer went back to the bio routine without involving multipath.c.? Perhaps we should be setting some end_io routine so that does happen, but I am not comfortable with the bio to request to request transitions taking place to go down that route.

But what was clear is - we shouldn't be returning an ABTS status. We should be returning a TRANSPORT_DISRUPTED status, so that multipath (nvme_complete_rq()) knows that it doesn't have to reset the controller (we're likely having errors for illegal state transitions if in connecting state), doesn't have to increment the retry counter, and can use it to possibly select an alternate path if desired.

-- james

#4 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

In reply to James Smart
Quoted 43 lines · James Smart

When an io is rejected by nvmf_check_ready() due to validation of the controller state, the nvmf_fail_nonready_command() will normally return BLK_STS_RESOURCE to requeue and retry. However, if the controller is dying or the io is marked for NVME multipath, the io is failed so that the controller can terminate or so that the io can be issued on a different path. Unfortunately, as this reject point is before the transport has accepted the command, blk-mq ends up completing the io and never calls nvme_complete_rq(), which is where multipath may preserve/re-route the io. The end result is, the device user ends up seeing an EIO error.

Example: single path connectivity, controller is under load, and a reset is induced. An i/o is received: a) while the reset state has been set but the queues have yet to be stopped; or b) after queues are started (at end of reset) but before the reconnect has completed. The io finishes with an EIO status.

This patch modifies the reject point such that it appears to queue successfully, but actually completes the io with an error and calls nvme_complete_rq().

Signed-off-by James Smart <jsmart2021 at gmail.com>
---
  drivers/nvme/host/fabrics.c | 11 ++++++++++-
  1 file changed, 10 insertions(+), 1 deletion(-)
drivers/nvme/host/fabrics.c +3 -1
index 206d63cb1afc..ed4b8e8e0249 100644
@@ -548,11 +548,20 @@ static struct nvmf_transport_ops *nvmf_lookup_transport(
  blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl,
  		struct request *rq)
  {
+	struct nvme_request *nvmerq = nvme_req(rq);
+
  	if (ctrl->state != NVME_CTRL_DELETING &&
  	    ctrl->state != NVME_CTRL_DEAD &&
  	    !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
  		return BLK_STS_RESOURCE;
-	nvme_req(rq)->status = NVME_SC_ABORT_REQ;
+

drivers/nvme/host/fabrics.c static struct nvmf_transport_ops *nvmf_lookup_transport( · line 567

Quoted 10 lines
drivers/nvme/host/fabrics.c +7 -0
+	nvmerq->status = NVME_SC_ABORT_REQ;
+	nvmerq->result.u64 = 0;
+	if (rq->cmd_flags & REQ_NVME_MPATH) {
+		blk_mq_start_request(rq);
+		nvme_complete_rq(rq);
+		return BLK_STS_OK;
+	}
  	return BLK_STS_IOERR;
  }
  EXPORT_SYMBOL_GPL(nvmf_fail_nonready_command);

Shouldn't we use '0x371' (as per TP 4028) for this? Description said 'Command Aborted By host: The command was aborted as a result of host action (e.g., the host disconnected the Fabric connection).' which seem to be a pretty good match ...

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)
#5 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

Quoted 7 lines · Hannes Reinecke
Quoted 3 lines

+??? nvmerq->status = NVME_SC_ABORT_REQ;

Shouldn't we use '0x371' (as per TP 4028) for this? Description said 'Command Aborted By host: The command was aborted as a result of host action (e.g., the host disconnected the Fabric connection).' which seem to be a pretty good match ...

Possibly - but...? If it was the transport that did a real io abort, it would make sense. But in this case, the io never even got started, so why mark it aborted ?? If it really was aborted, then doing a ctlr reset and incrementing retry counters may make sense. But if just bounced due to not being able to talk to the controller right now - those things should happen. So I still think we need a different value specific for the linux stack.

-- james

#6 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

Quoted 9 lines · James Smart
Quoted 7 lines · Hannes Reinecke
Quoted 3 lines

+??? nvmerq->status = NVME_SC_ABORT_REQ;

Shouldn't we use '0x371' (as per TP 4028) for this? Description said 'Command Aborted By host: The command was aborted as a result of host action (e.g., the host disconnected the Fabric connection).' which seem to be a pretty good match ...

Quoted 8 lines

Possibly - but...? If it was the transport that did a real io abort, it would make sense. But in this case, the io never even got started, so why mark it aborted ?? If it really was aborted, then doing a ctlr reset and incrementing retry counters may make sense. But if just bounced due to not being able to talk to the controller right now - those things should happen. So I still think we need a different value specific for the linux stack.

Hmm. Can we even define linux-only return values?
I hope we don't need to go creating an TPAR just for that ... But other than that I'm fine with use an internal return code here.

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)
#7 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

In reply to James Smart

On 9/21/18 8:49 PM, James Smart wrote:
[ .. ]

Quoted 12 lines
drivers/nvme/host/fabrics.c +2 -0
index 206d63cb1afc..ed4b8e8e0249 100644
@@ -548,11 +548,20 @@ static struct nvmf_transport_ops *nvmf_lookup_transport(
  blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl,
  		struct request *rq)
  {
+	struct nvme_request *nvmerq = nvme_req(rq);
+
  	if (ctrl->state != NVME_CTRL_DELETING &&
  	    ctrl->state != NVME_CTRL_DEAD &&

drivers/nvme/host/fabrics.c static struct nvmf_transport_ops *nvmf_lookup_transport( · line 563

Quoted 10 lines
drivers/nvme/host/fabrics.c +7 -1
  	    !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
  		return BLK_STS_RESOURCE;
-	nvme_req(rq)->status = NVME_SC_ABORT_REQ;
+
+	nvmerq->status = NVME_SC_ABORT_REQ;
+	nvmerq->result.u64 = 0;
+	if (rq->cmd_flags & REQ_NVME_MPATH) {
+		blk_mq_start_request(rq);
+		nvme_complete_rq(rq);
+		return BLK_STS_OK;

BLK_STS_TRANSPORT ?

drivers/nvme/host/fabrics.c static struct nvmf_transport_ops *nvmf_lookup_transport( · line 567

Quoted 5 lines
drivers/nvme/host/fabrics.c +1 -0
+	}
  	return BLK_STS_IOERR;
  }
  EXPORT_SYMBOL_GPL(nvmf_fail_nonready_command);

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)
#8 lore raw

PATCH 1/1 nvme: call nvme_complete_rq on NVME_MPATH failures

Quoted 19 lines · Hannes Reinecke

On 9/21/18 8:49 PM, James Smart wrote:
[ .. ]

Quoted 17 lines
drivers/nvme/host/fabrics.c +2 -0
index 206d63cb1afc..ed4b8e8e0249 100644
@@ -548,11 +548,20 @@ static struct nvmf_transport_ops 
 *nvmf_lookup_transport(
 ? blk_status_t nvmf_fail_nonready_command(struct nvme_ctrl *ctrl,
 ????????? struct request *rq)
 ? {
+??? struct nvme_request *nvmerq = nvme_req(rq);
+

????? if (ctrl->state != NVME_CTRL_DELETING &&
????????? ctrl->state != NVME_CTRL_DEAD &&
????????? !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH)) ????????? return BLK_STS_RESOURCE;
-??? nvme_req(rq)->status = NVME_SC_ABORT_REQ;
+

Quoted 9 lines
Quoted 6 lines

+??? nvmerq->status = NVME_SC_ABORT_REQ;
+??? nvmerq->result.u64 = 0;
+??? if (rq->cmd_flags & REQ_NVME_MPATH) {
+??????? blk_mq_start_request(rq);
+??????? nvme_complete_rq(rq);
+??????? return BLK_STS_OK;

BLK_STS_TRANSPORT ?

Ah, no. Not a good idea :-)

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)