Skip to contentSkip to conversation

More fixups for raid5

PATCH Applied Logan Gunthorpe 3 participants 9 messages Aug 11, 2022 – Aug 29, 2022 MD RAID

lore
Share & tools
lore
#0 lore raw

cover More fixups for raid5

Hey,

The first patch in this series is a fix for another race issue with the test infrastructure.

The remaining 4 patches address Christoph's feedback in my previous patchset that I sent rather late in the cycle. (Sorry about that).

This series is based on current md-next (ae0a80935d6a6).

Logan

--

David Sloan (1):
md: Flush workqueue md_rdev_misc_wq in md_alloc()

Logan Gunthorpe (4):
md/raid5: Refactor raid5_get_active_stripe() md/raid5: Drop extern on function declarations in raid5.h md/raid5: Cleanup prototype of raid5_get_active_stripe() md/raid5: Don't read ->active_stripes if it's not needed

 drivers/md/md.c          |   1 +
 drivers/md/raid5-cache.c |   3 +-
 drivers/md/raid5.c       | 132 ++++++++++++++++++++-------------------
 drivers/md/raid5.h       |  32 ++++++----
 4 files changed, 90 insertions(+), 78 deletions(-)

base-commit: ae0a80935d6a65764b0db00c8b03d3807b4110a6 --
2.30.2

#1 lore raw

PATCH 1/5 md: Flush workqueue md_rdev_misc_wq in md_alloc()

From: David Sloan <[email protected]>

A race condition still exists when removing and re-creating md devices in test cases. However, it is only seen on some setups.

The race condition was tracked down to a reference still being held to the kobject by the rdev in the md_rdev_misc_wq which will be released in rdev_delayed_delete().

md_alloc() waits for previous deletions by waiting on the md_misc_wq, but the md_rdev_misc_wq may still be holding a reference to a recently removed device.

To fix this, also flush the md_rdev_misc_wq in md_alloc().

---
 drivers/md/md.c | 1 +
 1 file changed, 1 insertion(+)
drivers/md/md.c +1 -0
index afaf36b2f6ab..71d221601bf8 100644
@@ -5620,6 +5620,7 @@ struct mddev *md_alloc(dev_t dev, char *name)
 	 * removed (mddev_delayed_delete).
 	 */
 	flush_workqueue(md_misc_wq);
+	flush_workqueue(md_rdev_misc_wq);
 
 	mutex_lock(&disks_mutex);
 	mddev = mddev_alloc(dev);
Signature · 1 line
2.30.2
maintainer
#2 lore raw

Re: [PATCH 1/5] md: Flush workqueue md_rdev_misc_wq in md_alloc()

Quoted 14 lines · Logan Gunthorpe

From: David Sloan <[email protected]>

A race condition still exists when removing and re-creating md devices in test cases. However, it is only seen on some setups.

The race condition was tracked down to a reference still being held to the kobject by the rdev in the md_rdev_misc_wq which will be released in rdev_delayed_delete().

md_alloc() waits for previous deletions by waiting on the md_misc_wq, but the md_rdev_misc_wq may still be holding a reference to a recently removed device.

Quoted 5 lines

To fix this, also flush the md_rdev_misc_wq in md_alloc().

Applied 1/5 to md-fixes.

Thanks!
Song

#3 lore raw

PATCH 2/5 md/raid5: Refactor raid5_get_active_stripe()

In reply to Logan Gunthorpe

Refactor raid5_get_active_stripe() without the gotos with an explicit infinite loop and some additional nesting.

---
 drivers/md/raid5.c | 82 +++++++++++++++++++++++-----------------------
 1 file changed, 41 insertions(+), 41 deletions(-)
drivers/md/raid5.c +41 -41
index 4456ac51f7c5..1288ef9e1571 100644
@@ -811,54 +811,54 @@ static struct stripe_head *__raid5_get_active_stripe(struct r5conf *conf,
 
 	spin_lock_irq(conf->hash_locks + hash);
 
-retry:
-	if (!noquiesce && conf->quiesce) {
-		/*
-		 * Must release the reference to batch_last before waiting,
-		 * on quiesce, otherwise the batch_last will hold a reference
-		 * to a stripe and raid5_quiesce() will deadlock waiting for
-		 * active_stripes to go to zero.
-		 */
-		if (ctx && ctx->batch_last) {
-			raid5_release_stripe(ctx->batch_last);
-			ctx->batch_last = NULL;
-		}
-
-		wait_event_lock_irq(conf->wait_for_quiescent, !conf->quiesce,
-				    *(conf->hash_locks + hash));
-	}
+	for (;;) {
+		if (!noquiesce && conf->quiesce) {
+			/*
+			 * Must release the reference to batch_last before
+			 * waiting, on quiesce, otherwise the batch_last will
+			 * hold a reference to a stripe and raid5_quiesce()
+			 * will deadlock waiting for active_stripes to go to
+			 * zero.
+			 */
+			if (ctx && ctx->batch_last) {
+				raid5_release_stripe(ctx->batch_last);
+				ctx->batch_last = NULL;
+			}
 
-	sh = find_get_stripe(conf, sector, conf->generation - previous, hash);
-	if (sh)
-		goto out;
+			wait_event_lock_irq(conf->wait_for_quiescent,
+					    !conf->quiesce,
+					    *(conf->hash_locks + hash));
+		}
 
-	if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
-		goto wait_for_stripe;
+		sh = find_get_stripe(conf, sector, conf->generation - previous,
+				     hash);
+		if (sh)
+			break;
 
-	sh = get_free_stripe(conf, hash);
-	if (sh) {
-		r5c_check_stripe_cache_usage(conf);
-		init_stripe(sh, sector, previous);
-		atomic_inc(&sh->count);
-		goto out;
-	}
+		if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
+			sh = get_free_stripe(conf, hash);
+			if (sh) {
+				r5c_check_stripe_cache_usage(conf);
+				init_stripe(sh, sector, previous);
+				atomic_inc(&sh->count);
+				break;
+			}
 
-	if (!test_bit(R5_DID_ALLOC, &conf->cache_state))
-		set_bit(R5_ALLOC_MORE, &conf->cache_state);
+			if (!test_bit(R5_DID_ALLOC, &conf->cache_state))
+				set_bit(R5_ALLOC_MORE, &conf->cache_state);
+		}
 
-wait_for_stripe:
-	if (noblock)
-		goto out;
+		if (noblock)
+			break;
 
-	set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
-	r5l_wake_reclaim(conf->log, 0);
-	wait_event_lock_irq(conf->wait_for_stripe,
-			    is_inactive_blocked(conf, hash),
-			    *(conf->hash_locks + hash));
-	clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
-	goto retry;
+		set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
+		r5l_wake_reclaim(conf->log, 0);
+		wait_event_lock_irq(conf->wait_for_stripe,
+				    is_inactive_blocked(conf, hash),
+				    *(conf->hash_locks + hash));
+		clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
+	}
 
-out:
 	spin_unlock_irq(conf->hash_locks + hash);
 	return sh;
 }
Signature · 1 line
2.30.2
maintainer Applied
#4 lore raw

Re: [PATCH 2/5] md/raid5: Refactor raid5_get_active_stripe()

Quoted 6 lines · Logan Gunthorpe

Refactor raid5_get_active_stripe() without the gotos with an explicit infinite loop and some additional nesting.

Applied to md-next.

Thanks!
Song

Quoted 107 lines
---
 drivers/md/raid5.c | 82 +++++++++++++++++++++++-----------------------
 1 file changed, 41 insertions(+), 41 deletions(-)
drivers/md/raid5.c +41 -42
index 4456ac51f7c5..1288ef9e1571 100644
@@ -811,54 +811,54 @@ static struct stripe_head *__raid5_get_active_stripe(struct r5conf *conf,
 
        spin_lock_irq(conf->hash_locks + hash);
 
-retry:
-       if (!noquiesce && conf->quiesce) {
-               /*
-                * Must release the reference to batch_last before waiting,
-                * on quiesce, otherwise the batch_last will hold a reference
-                * to a stripe and raid5_quiesce() will deadlock waiting for
-                * active_stripes to go to zero.
-                */
-               if (ctx && ctx->batch_last) {
-                       raid5_release_stripe(ctx->batch_last);
-                       ctx->batch_last = NULL;
-               }
-
-               wait_event_lock_irq(conf->wait_for_quiescent, !conf->quiesce,
-                                   *(conf->hash_locks + hash));
-       }
+       for (;;) {
+               if (!noquiesce && conf->quiesce) {
+                       /*
+                        * Must release the reference to batch_last before
+                        * waiting, on quiesce, otherwise the batch_last will
+                        * hold a reference to a stripe and raid5_quiesce()
+                        * will deadlock waiting for active_stripes to go to
+                        * zero.
+                        */
+                       if (ctx && ctx->batch_last) {
+                               raid5_release_stripe(ctx->batch_last);
+                               ctx->batch_last = NULL;
+                       }
 
-       sh = find_get_stripe(conf, sector, conf->generation - previous, hash);
-       if (sh)
-               goto out;
+                       wait_event_lock_irq(conf->wait_for_quiescent,
+                                           !conf->quiesce,
+                                           *(conf->hash_locks + hash));
+               }
 
-       if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
-               goto wait_for_stripe;
+               sh = find_get_stripe(conf, sector, conf->generation - previous,
+                                    hash);
+               if (sh)
+                       break;
 
-       sh = get_free_stripe(conf, hash);
-       if (sh) {
-               r5c_check_stripe_cache_usage(conf);
-               init_stripe(sh, sector, previous);
-               atomic_inc(&sh->count);
-               goto out;
-       }
+               if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
+                       sh = get_free_stripe(conf, hash);
+                       if (sh) {
+                               r5c_check_stripe_cache_usage(conf);
+                               init_stripe(sh, sector, previous);
+                               atomic_inc(&sh->count);
+                               break;
+                       }
 
-       if (!test_bit(R5_DID_ALLOC, &conf->cache_state))
-               set_bit(R5_ALLOC_MORE, &conf->cache_state);
+                       if (!test_bit(R5_DID_ALLOC, &conf->cache_state))
+                               set_bit(R5_ALLOC_MORE, &conf->cache_state);
+               }
 
-wait_for_stripe:
-       if (noblock)
-               goto out;
+               if (noblock)
+                       break;
 
-       set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
-       r5l_wake_reclaim(conf->log, 0);
-       wait_event_lock_irq(conf->wait_for_stripe,
-                           is_inactive_blocked(conf, hash),
-                           *(conf->hash_locks + hash));
-       clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
-       goto retry;
+               set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
+               r5l_wake_reclaim(conf->log, 0);
+               wait_event_lock_irq(conf->wait_for_stripe,
+                                   is_inactive_blocked(conf, hash),
+                                   *(conf->hash_locks + hash));
+               clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
+       }
 
-out:
        spin_unlock_irq(conf->hash_locks + hash);
        return sh;
 }
--

2.30.2

#5 lore raw

PATCH 3/5 md/raid5: Drop extern on function declarations in raid5.h

In reply to Logan Gunthorpe

externs should not be used in function declarations, so clean those up.

---
 drivers/md/raid5.h | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)
drivers/md/raid5.h +10 -12
index a5082bed83c8..4be2feb9e74a 100644
@@ -803,16 +803,14 @@ raid5_get_dev_page(struct stripe_head *sh, int disk_idx)
 }
 #endif
 
-extern void md_raid5_kick_device(struct r5conf *conf);
-extern int raid5_set_cache_size(struct mddev *mddev, int size);
-extern sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous);
-extern void raid5_release_stripe(struct stripe_head *sh);
-extern sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
-				     int previous, int *dd_idx,
-				     struct stripe_head *sh);
-extern struct stripe_head *
-raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
-			bool previous, bool noblock, bool noquiesce);
-extern int raid5_calc_degraded(struct r5conf *conf);
-extern int r5c_journal_mode_set(struct mddev *mddev, int journal_mode);
+void md_raid5_kick_device(struct r5conf *conf);
+int raid5_set_cache_size(struct mddev *mddev, int size);
+sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous);
+void raid5_release_stripe(struct stripe_head *sh);
+sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
+		int previous, int *dd_idx, struct stripe_head *sh);
+struct stripe_head *raid5_get_active_stripe(struct r5conf *conf,
+		sector_t sector, bool previous, bool noblock, bool noquiesce);
+int raid5_calc_degraded(struct r5conf *conf);
+int r5c_journal_mode_set(struct mddev *mddev, int journal_mode);
 #endif
Signature · 1 line
2.30.2
#6 lore raw

PATCH 4/5 md/raid5: Cleanup prototype of raid5_get_active_stripe()

In reply to Logan Gunthorpe

Drop the three bools in the prototype of raid5_get_active_stripe() and replace them with a flags parameter.

At the same time, drop the distinction with __raid5_get_active_stripe().

---
 drivers/md/raid5-cache.c |  3 ++-
 drivers/md/raid5.c       | 49 +++++++++++++++++++++-------------------
 drivers/md/raid5.h       | 12 +++++++++-
 3 files changed, 39 insertions(+), 25 deletions(-)
drivers/md/raid5-cache.c +2 -1
index f4e1cc1ece43..5e35a7321ded 100644
@@ -1923,7 +1923,8 @@ r5c_recovery_alloc_stripe(
 {
 	struct stripe_head *sh;
 
-	sh = raid5_get_active_stripe(conf, stripe_sect, 0, noblock, 0);
+	sh = raid5_get_active_stripe(conf, NULL, stripe_sect,
+				     noblock ? R5_GAS_NOBLOCK : 0);
 	if (!sh)
 		return NULL;  /* no more stripe available */
 
drivers/md/raid5.c +26 -23

132 lines not on this page · raw message

drivers/md/raid5.h +11 -1
index 4be2feb9e74a..e873938a6125 100644
@@ -809,8 +809,18 @@ sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous);
 void raid5_release_stripe(struct stripe_head *sh);
 sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
 		int previous, int *dd_idx, struct stripe_head *sh);
+
+struct stripe_request_ctx;
+/* get stripe from previous generation (when reshaping) */
+#define R5_GAS_PREVIOUS		(1 << 0)
+/* do not block waiting for a free stripe */
+#define R5_GAS_NOBLOCK		(1 << 1)
+/* do not block waiting for quiesce to be released */
+#define R5_GAS_NOQUIESCE	(1 << 2)
 struct stripe_head *raid5_get_active_stripe(struct r5conf *conf,
-		sector_t sector, bool previous, bool noblock, bool noquiesce);
+		struct stripe_request_ctx *ctx, sector_t sector,
+		unsigned int flags);
+
 int raid5_calc_degraded(struct r5conf *conf);
 int r5c_journal_mode_set(struct mddev *mddev, int journal_mode);
 #endif
Signature · 1 line
2.30.2
#7 lore raw

PATCH 5/5 md/raid5: Don't read ->active_stripes if it's not needed

In reply to Logan Gunthorpe

The atomic_read() is not needed in many cases so only do the read after the first checks are done.

---
 drivers/md/raid5.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
drivers/md/raid5.c +2 -3
index 7e7205bb40f1..0b73eeea1b19 100644
@@ -789,15 +789,14 @@ struct stripe_request_ctx {
  */
 static bool is_inactive_blocked(struct r5conf *conf, int hash)
 {
-	int active = atomic_read(&conf->active_stripes);
-
 	if (list_empty(conf->inactive_list + hash))
 		return false;
 
 	if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
 		return true;
 
-	return active < (conf->max_nr_stripes * 3 / 4);
+	return (atomic_read(&conf->active_stripes) <
+		(conf->max_nr_stripes * 3 / 4));
 }
 
 struct stripe_head *raid5_get_active_stripe(struct r5conf *conf,
Signature · 1 line
2.30.2