Skip to contentSkip to conversation

queue_rqs error handling

PATCH Applied Keith Busch 3 participants 9 messages Jan 5, 2022

Subsystems (3)
lore
Share & tools
lore

Keith Busch

author maintainer
#0 lore raw

v3 cover queue_rqs error handling

The only real change since v2 is a prep patch that relocates the rq list macros to blk-mq.h since that's where 'struct request' is defined.

Patch 3 removes the 'next' parameter since it is trivially obtainable via 'rq->rq_next' anyway.

Otherwise, the series is the same as v2 and tested with lots of random error injection in the prep path. The same errors would have lost requests in the current driver, but is successful with this series.

Keith Busch (4):
block: move rq_list macros to blk-mq.h
block: introduce rq_list_for_each_safe macro
block: introduce rq_list_move
nvme-pci: fix queue_rqs list splitting

 drivers/nvme/host/pci.c | 28 +++++++++++------------
 fs/io_uring.c           |  2 +-
 include/linux/blk-mq.h  | 50 +++++++++++++++++++++++++++++++++++++++++
 include/linux/blkdev.h  | 29 ------------------------
 4 files changed, 65 insertions(+), 44 deletions(-)
Signature · 1 line
2.25.4
author maintainer
#1 lore raw

PATCH v3 1/4 block: move rq_list macros to blk-mq.h

Move the request list macros to the header file that defines that struct they operate on.

---
 fs/io_uring.c          |  2 +-
 include/linux/blk-mq.h | 29 +++++++++++++++++++++++++++++
 include/linux/blkdev.h | 29 -----------------------------
 3 files changed, 30 insertions(+), 30 deletions(-)
fs/io_uring.c +1 -1
index 90002bb3fdf4..33f72b3b302c 100644
@@ -57,7 +57,7 @@
 #include <linux/mman.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
-#include <linux/blkdev.h>
+#include <linux/blk-mq.h>
 #include <linux/bvec.h>
 #include <linux/net.h>
 #include <net/sock.h>
include/linux/blk-mq.h +29 -0
index 550996cf419c..bf64b94cd64e 100644
@@ -216,6 +216,35 @@ static inline unsigned short req_get_ioprio(struct request *req)
 #define rq_dma_dir(rq) \
 	(op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
 
+#define rq_list_add(listptr, rq)	do {		\
+	(rq)->rq_next = *(listptr);			\
+	*(listptr) = rq;				\
+} while (0)
+
+#define rq_list_pop(listptr)				\
+({							\
+	struct request *__req = NULL;			\
+	if ((listptr) && *(listptr))	{		\
+		__req = *(listptr);			\
+		*(listptr) = __req->rq_next;		\
+	}						\
+	__req;						\
+})
+
+#define rq_list_peek(listptr)				\
+({							\
+	struct request *__req = NULL;			\
+	if ((listptr) && *(listptr))			\
+		__req = *(listptr);			\
+	__req;						\
+})
+
+#define rq_list_for_each(listptr, pos)			\
+	for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos))
+
+#define rq_list_next(rq)	(rq)->rq_next
+#define rq_list_empty(list)	((list) == (struct request *) NULL)
+
 enum blk_eh_timer_return {
 	BLK_EH_DONE,		/* drivers has completed the command */
 	BLK_EH_RESET_TIMER,	/* reset timer and try again */
include/linux/blkdev.h +0 -29
index 22746b2d6825..9c95df26fc26 100644
@@ -1339,33 +1339,4 @@ struct io_comp_batch {
 
 #define DEFINE_IO_COMP_BATCH(name)	struct io_comp_batch name = { }
 
-#define rq_list_add(listptr, rq)	do {		\
-	(rq)->rq_next = *(listptr);			\
-	*(listptr) = rq;				\
-} while (0)
-
-#define rq_list_pop(listptr)				\
-({							\
-	struct request *__req = NULL;			\
-	if ((listptr) && *(listptr))	{		\
-		__req = *(listptr);			\
-		*(listptr) = __req->rq_next;		\
-	}						\
-	__req;						\
-})
-
-#define rq_list_peek(listptr)				\
-({							\
-	struct request *__req = NULL;			\
-	if ((listptr) && *(listptr))			\
-		__req = *(listptr);			\
-	__req;						\
-})
-
-#define rq_list_for_each(listptr, pos)			\
-	for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos))
-
-#define rq_list_next(rq)	(rq)->rq_next
-#define rq_list_empty(list)	((list) == (struct request *) NULL)
-
 #endif /* _LINUX_BLKDEV_H */
Signature · 1 line
2.25.4
author maintainer
#3 lore raw

PATCH v3 2/4 block: introduce rq_list_for_each_safe macro

In reply to Keith Busch

While iterating a list, a particular request may need to be removed for special handling. Provide an iterator that can safely handle that.

---
 include/linux/blk-mq.h | 4 ++++
 1 file changed, 4 insertions(+)
include/linux/blk-mq.h +4 -0
index bf64b94cd64e..1467f0fa2142 100644
@@ -242,6 +242,10 @@ static inline unsigned short req_get_ioprio(struct request *req)
 #define rq_list_for_each(listptr, pos)			\
 	for (pos = rq_list_peek((listptr)); pos; pos = rq_list_next(pos))
 
+#define rq_list_for_each_safe(listptr, pos, nxt)			\
+	for (pos = rq_list_peek((listptr)), nxt = rq_list_next(pos);	\
+		pos; pos = nxt, nxt = pos ? rq_list_next(pos) : NULL)
+
 #define rq_list_next(rq)	(rq)->rq_next
 #define rq_list_empty(list)	((list) == (struct request *) NULL)
 
Signature · 1 line
2.25.4
author maintainer
#4 lore raw

PATCH v3 3/4 block: introduce rq_list_move

In reply to Keith Busch

When iterating a list, a particular request may need to be moved for special handling. Provide a helper function to achieve that so drivers don't need to reimplement rqlist manipulation.

---
 include/linux/blk-mq.h | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
include/linux/blk-mq.h +17 -0
index 1467f0fa2142..f40a05ecca4a 100644
@@ -249,6 +249,23 @@ static inline unsigned short req_get_ioprio(struct request *req)
 #define rq_list_next(rq)	(rq)->rq_next
 #define rq_list_empty(list)	((list) == (struct request *) NULL)
 
+/**
+ * rq_list_move() - move a struct request from one list to another
+ * @src: The source list @rq is currently in
+ * @dst: The destination list that @rq will be appended to
+ * @rq: The request to move
+ * @prev: The request preceding @rq in @src (NULL if @rq is the head)
+ */
+static void inline rq_list_move(struct request **src, struct request **dst,
+				struct request *rq, struct request *prev)
+{
+	if (prev)
+		prev->rq_next = rq->rq_next;
+	else
+		*src = rq->rq_next;
+	rq_list_add(dst, rq);
+}
+
 enum blk_eh_timer_return {
 	BLK_EH_DONE,		/* drivers has completed the command */
 	BLK_EH_RESET_TIMER,	/* reset timer and try again */
Signature · 1 line
2.25.4
author maintainer
#6 lore raw

PATCH v3 4/4 nvme-pci: fix queue_rqs list splitting

In reply to Keith Busch

If command prep fails, current handling will orphan subsequent requests in the list. Consider a simple example:

rqlist = [ 1 -> 2 ]

When prep for request '1' fails, it will be appended to the 'requeue_list', leaving request '2' disconnected from the original rqlist and no longer tracked. Meanwhile, rqlist is still pointing to the failed request '1' and will attempt to submit the unprepped command.

Fix this by updating the rqlist accordingly using the request list helper functions.

Fixes d62cbcf62f2f ("nvme: add support for mq_ops->queue_rqs()")
---
 drivers/nvme/host/pci.c | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)
drivers/nvme/host/pci.c +14 -14
index 50deb8b69c40..d8585df2c2fd 100644
@@ -999,30 +999,30 @@ static bool nvme_prep_rq_batch(struct nvme_queue *nvmeq, struct request *req)
 
 static void nvme_queue_rqs(struct request **rqlist)
 {
-	struct request *req = rq_list_peek(rqlist), *prev = NULL;
+	struct request *req, *next, *prev = NULL;
 	struct request *requeue_list = NULL;
 
-	do {
+	rq_list_for_each_safe(rqlist, req, next) {
 		struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 
 		if (!nvme_prep_rq_batch(nvmeq, req)) {
 			/* detach 'req' and add to remainder list */
-			if (prev)
-				prev->rq_next = req->rq_next;
-			rq_list_add(&requeue_list, req);
-		} else {
-			prev = req;
+			rq_list_move(rqlist, &requeue_list, req, prev);
+
+			req = prev;
+			if (!req)
+				continue;
 		}
 
-		req = rq_list_next(req);
-		if (!req || (prev && req->mq_hctx != prev->mq_hctx)) {
+		if (!next || req->mq_hctx != next->mq_hctx) {
 			/* detach rest of list, and submit */
-			if (prev)
-				prev->rq_next = NULL;
+			req->rq_next = NULL;
 			nvme_submit_cmds(nvmeq, rqlist);
-			*rqlist = req;
-		}
-	} while (req);
+			*rqlist = next;
+			prev = NULL;
+		} else
+			prev = req;
+	}
 
 	*rqlist = requeue_list;
 }
Signature · 1 line
2.25.4
maintainer Reviewed-by on v3 4/4
#7 lore raw

Re: [PATCHv3 4/4] nvme-pci: fix queue_rqs list splitting

Quoted 10 lines · Keith Busch

If command prep fails, current handling will orphan subsequent requests in the list. Consider a simple example:

rqlist = [ 1 -> 2 ]

When prep for request '1' fails, it will be appended to the 'requeue_list', leaving request '2' disconnected from the original rqlist and no longer tracked. Meanwhile, rqlist is still pointing to the failed request '1' and will attempt to submit the unprepped command.

Quoted 5 lines

Fix this by updating the rqlist accordingly using the request list helper functions.

Fixes d62cbcf62f2f ("nvme: add support for mq_ops->queue_rqs()")

Looks good,

maintainer Applied
#8 lore raw
In reply to Keith Busch
Quoted 11 lines · Keith Busch

The only real change since v2 is a prep patch that relocates the rq list macros to blk-mq.h since that's where 'struct request' is defined.

Patch 3 removes the 'next' parameter since it is trivially obtainable via 'rq->rq_next' anyway.

Otherwise, the series is the same as v2 and tested with lots of random error injection in the prep path. The same errors would have lost requests in the current driver, but is successful with this series.

[...]

Applied, thanks!

[1/4] block: move rq_list macros to blk-mq.h

      commit: edce22e19bfa86efa2522d041d6367f2f099e8ed
[2/4] block: introduce rq_list_for_each_safe macro
      commit: 3764fd05e1f89530e2ee5cbff0b638f2b1141b90
[3/4] block: introduce rq_list_move
      commit: d2528be7a8b09af9796a270debd14101a72bb552
[4/4] nvme-pci: fix queue_rqs list splitting
      commit: 6bfec7992ec79b63fb07330ae97f3fb43120aa37

Best regards,

Signature · 1 line
Jens Axboe