blk-mq: blk-mq: provide forced completion method

Use blk_mq_foce_complete_rq() to bypass fake timeout error injection so that request reclaim may proceed.

Reviewed-by Johannes Thumshirn <[email protected]>

Reviewed-by Ming Lei <[email protected]>

Signed-off-by Keith Busch <[email protected]>

Drivers may need to bypass error injection for error recovery. Rename __blk_mq_complete_request() to blk_mq_force_complete_rq() and export that function so drivers may skip potential fake timeouts after they've reclaimed lost requests.

Signed-off-by Keith Busch <[email protected]>

Drivers may need to bypass error injection for error recovery. Rename __blk_mq_complete_request() to blk_mq_force_complete_rq() and export that function so drivers may skip potential fake timeouts after they've reclaimed lost requests.

Drivers may need to bypass error injection for error recovery. Rename __blk_mq_complete_request() to blk_mq_force_complete_rq() and export that function so drivers may skip potential fake timeouts after they've reclaimed lost requests.

Signed-off-by Keith Busch <[email protected]>

---
  block/blk-mq.c         | 15 +++++++++++++--
  include/linux/blk-mq.h |  1 +
  2 files changed, 14 insertions(+), 2 deletions(-)

block/blk-mq.c +13 -2

index cac11945f602..560a114a82f8 100644
@@ -556,7 +556,17 @@ static void __blk_mq_complete_request_remote(void *data)
  	q->mq_ops->complete(rq);
  }
 
-static void __blk_mq_complete_request(struct request *rq)
+/**
+ * blk_mq_force_complete_rq() - Force complete the request, bypassing any error
+ * 				injection that could drop the completion.
+ * @rq: Request to be force completed
+ *
+ * Drivers should use blk_mq_complete_request() to complete requests in their
+ * normal IO path. For timeout error recovery, drivers may call this forced
+ * completion routine after they've reclaimed timed out requests to bypass
+ * potentially subsequent fake timeouts.
+ */
+void blk_mq_force_complete_rq(struct request *rq)
  {
  	struct blk_mq_ctx *ctx = rq->mq_ctx;
  	struct request_queue *q = rq->q;

@@ -602,6 +612,7 @@ static void __blk_mq_complete_request(struct request *rq)
  	}
  	put_cpu();
  }
+EXPORT_SYMBOL_GPL(blk_mq_force_complete_rq);
 
  static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
  	__releases(hctx->srcu)

@@ -635,7 +646,7 @@ bool blk_mq_complete_request(struct request *rq)
  {
  	if (unlikely(blk_should_fake_timeout(rq->q)))
  		return false;
-	__blk_mq_complete_request(rq);
+	blk_mq_force_complete_rq(rq);
  	return true;
  }
  EXPORT_SYMBOL(blk_mq_complete_request);

include/linux/blk-mq.h +1 -0

index d7307795439a..856bb10993cf 100644
@@ -494,6 +494,7 @@ void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list);
  void blk_mq_kick_requeue_list(struct request_queue *q);
  void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs);
  bool blk_mq_complete_request(struct request *rq);
+void blk_mq_force_complete_rq(struct request *rq);
  bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
  			   struct bio *bio, unsigned int nr_segs);
  bool blk_mq_queue_stopped(struct request_queue *q);