Fix kyber hang

PATCH In review Jens Axboe 3 participants 7 messages Dec 20, 2018

Subsystems (2)

Block layer Library code

lore

Share & tools

lore

Jens Axboe

author maintainer

2018-12-20 16:01 UTC #0 lore raw

PATCH v2 1/1 Fix kyber hang

Since the wrong kyber patch was included in the first one, here's a v2 with just that one line corrected.

First one adds the necessary sbitmap helpers for users to be able to do add_wait_queue() and list_del(wait_entry) type operations, and still work with the optimized wakeup checking.

Second one converts kyber to use it, fixing a hang with domain tokens.

Signature · 1 line

Jens Axboe

Jens Axboe

author maintainer

2018-12-20 16:01 UTC #1 lore raw

PATCH 1/2 sbitmap: add helpers for add/del wait queue handling

After commit 5d2ee7122c73, users of sbitmap that need wait queue handling must use the provided helpers. But we only added prepare_to_wait()/finish_wait() style helpers, add the equivalent add_wait_queue/list_del wrappers as we..

This is needed to ensure kyber plays by the sbitmap waitqueue rules.

Signed-off-by Jens Axboe <[email protected]>

---
 include/linux/sbitmap.h | 16 ++++++++++++++--
 lib/sbitmap.c           | 30 ++++++++++++++++++++++++++----
 2 files changed, 40 insertions(+), 6 deletions(-)

include/linux/sbitmap.h +14 -2

index 03f50fcedc79..14d558146aea 100644
@@ -560,13 +560,13 @@ void sbitmap_queue_wake_up(struct sbitmap_queue *sbq);
 void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m);
 
 struct sbq_wait {
-	int accounted;
+	struct sbitmap_queue *sbq;	/* if set, sbq_wait is accounted */
 	struct wait_queue_entry wait;
 };
 
 #define DEFINE_SBQ_WAIT(name)							\
 	struct sbq_wait name = {						\
-		.accounted = 0,							\
+		.sbq = NULL,							\
 		.wait = {							\
 			.private	= current,				\
 			.func		= autoremove_wake_function,		\

@@ -588,4 +588,16 @@ void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
 void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
 				struct sbq_wait *sbq_wait);
 
+/*
+ * Wrapper around add_wait_queue(), which maintains some extra internal state
+ */
+void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
+			    struct sbq_wait_state *ws,
+			    struct sbq_wait *sbq_wait);
+
+/*
+ * Must be paired with sbitmap_add_wait_queue()
+ */
+void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait);
+
 #endif /* __LINUX_SCALE_BITMAP_H */

lib/sbitmap.c +26 -4

index 5b3e56d68dab..65c2d06250a6 100644
@@ -671,13 +671,35 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m)
 }
 EXPORT_SYMBOL_GPL(sbitmap_queue_show);
 
+void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
+			    struct sbq_wait_state *ws,
+			    struct sbq_wait *sbq_wait)
+{
+	if (!sbq_wait->sbq) {
+		sbq_wait->sbq = sbq;
+		atomic_inc(&sbq->ws_active);
+	}
+	add_wait_queue(&ws->wait, &sbq_wait->wait);
+}
+EXPORT_SYMBOL_GPL(sbitmap_add_wait_queue);
+
+void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait)
+{
+	list_del_init(&sbq_wait->wait.entry);
+	if (sbq_wait->sbq) {
+		atomic_dec(&sbq_wait->sbq->ws_active);
+		sbq_wait->sbq = NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(sbitmap_del_wait_queue);
+
 void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
 			     struct sbq_wait_state *ws,
 			     struct sbq_wait *sbq_wait, int state)
 {
-	if (!sbq_wait->accounted) {
+	if (!sbq_wait->sbq) {
 		atomic_inc(&sbq->ws_active);
-		sbq_wait->accounted = 1;
+		sbq_wait->sbq = sbq;
 	}
 	prepare_to_wait_exclusive(&ws->wait, &sbq_wait->wait, state);
 }

@@ -687,9 +709,9 @@ void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
 			 struct sbq_wait *sbq_wait)
 {
 	finish_wait(&ws->wait, &sbq_wait->wait);
-	if (sbq_wait->accounted) {
+	if (sbq_wait->sbq) {
 		atomic_dec(&sbq->ws_active);
-		sbq_wait->accounted = 0;
+		sbq_wait->sbq = NULL;
 	}
 }
 EXPORT_SYMBOL_GPL(sbitmap_finish_wait);

Signature · 1 line

2.17.1

Jens Axboe

author maintainer

2018-12-20 16:01 UTC #2 lore raw

PATCH 2/2 kyber: use sbitmap add_wait_queue/list_del wait helpers

In reply to Jens Axboe

sbq_wake_ptr() checks sbq->ws_active to know if it needs to loop the wait indexes or not. This requires the use of the sbitmap waitqueue wrappers, but kyber doesn't use those for its domain token waitqueue handling.

Convert kyber to use the helpers. This fixes a hang with waiting for domain tokens.

Fixes 5d2ee7122c73 ("sbitmap: optimize wakeup check")

Reported-by Ming Lei <[email protected]>

Signed-off-by Jens Axboe <[email protected]>

---
 block/kyber-iosched.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

block/kyber-iosched.c +15 ~~-13~~

index de78e8aa7b0a..ec6a04e01bc1 100644
@@ -195,7 +195,7 @@ struct kyber_hctx_data {
 	unsigned int batching;
 	struct kyber_ctx_queue *kcqs;
 	struct sbitmap kcq_map[KYBER_NUM_DOMAINS];
-	wait_queue_entry_t domain_wait[KYBER_NUM_DOMAINS];
+	struct sbq_wait domain_wait[KYBER_NUM_DOMAINS];
 	struct sbq_wait_state *domain_ws[KYBER_NUM_DOMAINS];
 	atomic_t wait_index[KYBER_NUM_DOMAINS];
 };

@@ -501,10 +501,11 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 
 	for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
 		INIT_LIST_HEAD(&khd->rqs[i]);
-		init_waitqueue_func_entry(&khd->domain_wait[i],
+		khd->domain_wait[i].sbq = NULL;
+		init_waitqueue_func_entry(&khd->domain_wait[i].wait,
 					  kyber_domain_wake);
-		khd->domain_wait[i].private = hctx;
-		INIT_LIST_HEAD(&khd->domain_wait[i].entry);
+		khd->domain_wait[i].wait.private = hctx;
+		INIT_LIST_HEAD(&khd->domain_wait[i].wait.entry);
 		atomic_set(&khd->wait_index[i], 0);
 	}

@@ -698,12 +699,13 @@ static void kyber_flush_busy_kcqs(struct kyber_hctx_data *khd,
 			     flush_busy_kcq, &data);
 }
 
-static int kyber_domain_wake(wait_queue_entry_t *wait, unsigned mode, int flags,
+static int kyber_domain_wake(wait_queue_entry_t *wqe, unsigned mode, int flags,
 			     void *key)
 {
-	struct blk_mq_hw_ctx *hctx = READ_ONCE(wait->private);
+	struct blk_mq_hw_ctx *hctx = READ_ONCE(wqe->private);
+	struct sbq_wait *wait = container_of(wqe, struct sbq_wait, wait);
 
-	list_del_init(&wait->entry);
+	sbitmap_del_wait_queue(wait);
 	blk_mq_run_hw_queue(hctx, true);
 	return 1;
 }

@@ -714,7 +716,7 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
 {
 	unsigned int sched_domain = khd->cur_domain;
 	struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];
-	wait_queue_entry_t *wait = &khd->domain_wait[sched_domain];
+	struct sbq_wait *wait = &khd->domain_wait[sched_domain];
 	struct sbq_wait_state *ws;
 	int nr;

@@ -725,11 +727,11 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
 	 * run when one becomes available. Note that this is serialized on
 	 * khd->lock, but we still need to be careful about the waker.
 	 */
-	if (nr < 0 && list_empty_careful(&wait->entry)) {
+	if (nr < 0 && list_empty_careful(&wait->wait.entry)) {
 		ws = sbq_wait_ptr(domain_tokens,
 				  &khd->wait_index[sched_domain]);
 		khd->domain_ws[sched_domain] = ws;
-		add_wait_queue(&ws->wait, wait);
+		sbitmap_add_wait_queue(domain_tokens, ws, wait);
 
 		/*
 		 * Try again in case a token was freed before we got on the wait

@@ -745,10 +747,10 @@ static int kyber_get_domain_token(struct kyber_queue_data *kqd,
 	 * between the !list_empty_careful() check and us grabbing the lock, but
 	 * list_del_init() is okay with that.
 	 */
-	if (nr >= 0 && !list_empty_careful(&wait->entry)) {
+	if (nr >= 0 && !list_empty_careful(&wait->wait.entry)) {
 		ws = khd->domain_ws[sched_domain];
 		spin_lock_irq(&ws->wait.lock);
-		list_del_init(&wait->entry);
+		sbitmap_del_wait_queue(wait);
 		spin_unlock_irq(&ws->wait.lock);
 	}

@@ -951,7 +953,7 @@ static int kyber_##name##_waiting_show(void *data, struct seq_file *m)	\
 {									\
 	struct blk_mq_hw_ctx *hctx = data;				\
 	struct kyber_hctx_data *khd = hctx->sched_data;			\
-	wait_queue_entry_t *wait = &khd->domain_wait[domain];		\
+	wait_queue_entry_t *wait = &khd->domain_wait[domain].wait;	\
 									\
 	seq_printf(m, "%d\n", !list_empty_careful(&wait->entry));	\
 	return 0;							\

Signature · 1 line

2.17.1

Ming Lei

Tested-by on v2 1/1

2018-12-20 16:56 UTC #3 lore raw

In reply to Jens Axboe

Quoted 8 lines · Jens Axboe

Since the wrong kyber patch was included in the first one, here's a v2 with just that one line corrected.
First one adds the necessary sbitmap helpers for users to be able to do add_wait_queue() and list_del(wait_entry) type operations, and still work with the optimized wakeup checking.
Second one converts kyber to use it, fixing a hang with domain tokens.

With the two patches, can't reproduce the io hang issue any more when kyber is used.

Tested-by Ming Lei <[email protected]>

Thanks,
Ming

Jens Axboe

author maintainer

2018-12-20 17:37 UTC #4 lore raw

Quoted 8 lines · Ming Lei

Quoted 7 lines · Jens Axboe
Since the wrong kyber patch was included in the first one, here's a v2 with just that one line corrected.
First one adds the necessary sbitmap helpers for users to be able to do add_wait_queue() and list_del(wait_entry) type operations, and still work with the optimized wakeup checking.

Quoted 6 lines

Quoted 1 line
Second one converts kyber to use it, fixing a hang with domain tokens.
With the two patches, can't reproduce the io hang issue any more when kyber is used.
Tested-by Ming Lei <[email protected]>

Awesome, thanks for reporting and testing the patches!

Signature · 1 line

Jens Axboe

Omar Sandoval

Reviewed-by on v2 1/1

2018-12-20 19:15 UTC #5 lore raw

In reply to Jens Axboe

Quoted 8 lines · Jens Axboe

Since the wrong kyber patch was included in the first one, here's a v2 with just that one line corrected.
First one adds the necessary sbitmap helpers for users to be able to do add_wait_queue() and list_del(wait_entry) type operations, and still work with the optimized wakeup checking.
Second one converts kyber to use it, fixing a hang with domain tokens.

Lame, I should've caught this on the original patch. For the series:

Reviewed-by Omar Sandoval <[email protected]>

Jens Axboe

author maintainer

2018-12-20 19:16 UTC #6 lore raw

Quoted 11 lines · Omar Sandoval

Quoted 8 lines · Jens Axboe
Since the wrong kyber patch was included in the first one, here's a v2 with just that one line corrected.
First one adds the necessary sbitmap helpers for users to be able to do add_wait_queue() and list_del(wait_entry) type operations, and still work with the optimized wakeup checking.
Second one converts kyber to use it, fixing a hang with domain tokens.
Lame, I should've caught this on the original patch. For the series:

Yeah, somewhat puzzled that it wasn't...

Quoted 1 line

Reviewed-by Omar Sandoval <[email protected]>

Thanks!

Signature · 1 line

Jens Axboe