Skip to contentSkip to conversation

md/raid5: fix atomicity violation in raid5_cache_count

PATCH Superseded Gui-Dong Han 2 participants 3 messages Jan 11, 2024 – Jan 12, 2024 MD RAID

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 md/raid5: fix atomicity violation in raid5_cache_count

In raid5_cache_count():

    if (conf->max_nr_stripes < conf->min_nr_stripes)
        return 0;
    return conf->max_nr_stripes - conf->min_nr_stripes;

The current check is ineffective, as the values could change immediately after being checked.

In raid5_set_cache_size():

    ...
    conf->min_nr_stripes = size;
    ...
    while (size > conf->max_nr_stripes)
        conf->min_nr_stripes = conf->max_nr_stripes;
    ...

Due to intermediate value updates in raid5_set_cache_size(), concurrent execution of raid5_cache_count() and raid5_set_cache_size() may lead to inconsistent reads of conf->max_nr_stripes and conf->min_nr_stripes. The current checks are ineffective as values could change immediately after being checked, raising the risk of conf->min_nr_stripes exceeding conf->max_nr_stripes and potentially causing an integer overflow.

This possible bug is found by an experimental static analysis tool developed by our team. This tool analyzes the locking APIs to extract function pairs that can be concurrently executed, and then analyzes the instructions in the paired functions to identify possible concurrency bugs including data races and atomicity violations. The above possible bug is reported when our tool analyzes the source code of Linux 6.2.

To resolve this issue, it is suggested to introduce local variables 'min_stripes' and 'max_stripes' in raid5_cache_count() to ensure the values remain stable throughout the check. Adding locks in raid5_cache_count() fails to resolve atomicity violations, as raid5_set_cache_size() may hold intermediate values of conf->min_nr_stripes while unlocked. With this patch applied, our tool no longer reports the bug, with the kernel configuration allyesconfig for x86_64. Due to the lack of associated hardware, we cannot test the patch in runtime testing, and just verify it according to the code logic.

Fixes edbe83ab4c27 ("md/raid5: allow the stripe_cache to grow and ...")

---
v2:
* In this patch v2, we've updated to use READ_ONCE() instead of direct reads for accessing max_nr_stripes and min_nr_stripes, since read and write can concurrent.
Thank Yu Kuai for helpful advice.
---
v3:
* In this patch v3, we've updated to use WRITE_ONCE() in raid5_set_cache_size(), grow_one_stripe() and drop_one_stripe(), in order to pair READ_ONCE() with WRITE_ONCE().
Thank Yu Kuai for helpful advice.

---
 drivers/md/raid5.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)
drivers/md/raid5.c +8 -6
index 26e1e8a5e941..ece2b2094dc0 100644
@@ -2422,7 +2422,7 @@ static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
 	atomic_inc(&conf->active_stripes);
 
 	raid5_release_stripe(sh);
-	conf->max_nr_stripes++;
+	WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes + 1);
 	return 1;
 }
 
@@ -2717,7 +2717,7 @@ static int drop_one_stripe(struct r5conf *conf)
 	shrink_buffers(sh);
 	free_stripe(conf->slab_cache, sh);
 	atomic_dec(&conf->active_stripes);
-	conf->max_nr_stripes--;
+	WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes - 1);
 	return 1;
 }
 
@@ -6878,7 +6878,7 @@ raid5_set_cache_size(struct mddev *mddev, int size)
 	if (size <= 16 || size > 32768)
 		return -EINVAL;
 
-	conf->min_nr_stripes = size;
+	WRITE_ONCE(conf->min_nr_stripes, size);
 	mutex_lock(&conf->cache_size_mutex);
 	while (size < conf->max_nr_stripes &&
 	       drop_one_stripe(conf))
@@ -6890,7 +6890,7 @@ raid5_set_cache_size(struct mddev *mddev, int size)
 	mutex_lock(&conf->cache_size_mutex);
 	while (size > conf->max_nr_stripes)
 		if (!grow_one_stripe(conf, GFP_KERNEL)) {
-			conf->min_nr_stripes = conf->max_nr_stripes;
+			WRITE_ONCE(conf->min_nr_stripes, conf->max_nr_stripes);
 			result = -ENOMEM;
 			break;
 		}
@@ -7449,10 +7449,12 @@ static unsigned long raid5_cache_count(struct shrinker *shrink,
 {
 	struct r5conf *conf = shrink->private_data;
 
-	if (conf->max_nr_stripes < conf->min_nr_stripes)
+	int max_stripes = READ_ONCE(conf->max_nr_stripes);
+	int min_stripes = READ_ONCE(conf->min_nr_stripes);
+	if (max_stripes < min_stripes)
 		/* unlikely, but not impossible */
 		return 0;
-	return conf->max_nr_stripes - conf->min_nr_stripes;
+	return max_stripes - min_stripes;
 }
 
 static struct r5conf *setup_conf(struct mddev *mddev)
Signature · 1 line
2.34.1
#1 lore raw
Quoted 29 lines · Gui-Dong Han

In raid5_cache_count():

     if (conf->max_nr_stripes < conf->min_nr_stripes)
         return 0;
     return conf->max_nr_stripes - conf->min_nr_stripes;

The current check is ineffective, as the values could change immediately after being checked.

In raid5_set_cache_size():

     ...
     conf->min_nr_stripes = size;
     ...
     while (size > conf->max_nr_stripes)
         conf->min_nr_stripes = conf->max_nr_stripes;
     ...

Due to intermediate value updates in raid5_set_cache_size(), concurrent execution of raid5_cache_count() and raid5_set_cache_size() may lead to inconsistent reads of conf->max_nr_stripes and conf->min_nr_stripes. The current checks are ineffective as values could change immediately after being checked, raising the risk of conf->min_nr_stripes exceeding conf->max_nr_stripes and potentially causing an integer overflow.

This possible bug is found by an experimental static analysis tool developed by our team. This tool analyzes the locking APIs to extract function pairs that can be concurrently executed, and then analyzes the instructions in the paired functions to identify possible concurrency bugs including data races and atomicity violations. The above possible bug is reported when our tool analyzes the source code of Linux 6.2.

Quoted 10 lines

To resolve this issue, it is suggested to introduce local variables 'min_stripes' and 'max_stripes' in raid5_cache_count() to ensure the values remain stable throughout the check. Adding locks in raid5_cache_count() fails to resolve atomicity violations, as raid5_set_cache_size() may hold intermediate values of conf->min_nr_stripes while unlocked. With this patch applied, our tool no longer reports the bug, with the kernel configuration allyesconfig for x86_64. Due to the lack of associated hardware, we cannot test the patch in runtime testing, and just verify it according to the code logic.

This patch LGTM, just one nit below.

Quoted 60 lines
Fixes edbe83ab4c27 ("md/raid5: allow the stripe_cache to grow and ...")

---
v2:
* In this patch v2, we've updated to use READ_ONCE() instead of direct reads for accessing max_nr_stripes and min_nr_stripes, since read and write can concurrent.
Thank Yu Kuai for helpful advice.
---
v3:
* In this patch v3, we've updated to use WRITE_ONCE() in raid5_set_cache_size(), grow_one_stripe() and drop_one_stripe(), in order to pair READ_ONCE() with WRITE_ONCE().
Thank Yu Kuai for helpful advice.

---
  drivers/md/raid5.c | 14 ++++++++------
  1 file changed, 8 insertions(+), 6 deletions(-)
drivers/md/raid5.c +4 -4
index 26e1e8a5e941..ece2b2094dc0 100644
@@ -2422,7 +2422,7 @@ static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
  	atomic_inc(&conf->active_stripes);
 
  	raid5_release_stripe(sh);
-	conf->max_nr_stripes++;
+	WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes + 1);
  	return 1;
  }
 
@@ -2717,7 +2717,7 @@ static int drop_one_stripe(struct r5conf *conf)
  	shrink_buffers(sh);
  	free_stripe(conf->slab_cache, sh);
  	atomic_dec(&conf->active_stripes);
-	conf->max_nr_stripes--;
+	WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes - 1);
  	return 1;
  }
 
@@ -6878,7 +6878,7 @@ raid5_set_cache_size(struct mddev *mddev, int size)
  	if (size <= 16 || size > 32768)
  		return -EINVAL;
 
-	conf->min_nr_stripes = size;
+	WRITE_ONCE(conf->min_nr_stripes, size);
  	mutex_lock(&conf->cache_size_mutex);
  	while (size < conf->max_nr_stripes &&
  	       drop_one_stripe(conf))
@@ -6890,7 +6890,7 @@ raid5_set_cache_size(struct mddev *mddev, int size)
  	mutex_lock(&conf->cache_size_mutex);
  	while (size > conf->max_nr_stripes)
  		if (!grow_one_stripe(conf, GFP_KERNEL)) {
-			conf->min_nr_stripes = conf->max_nr_stripes;
+			WRITE_ONCE(conf->min_nr_stripes, conf->max_nr_stripes);
  			result = -ENOMEM;
  			break;
  		}

drivers/md/raid5.c static unsigned long raid5_cache_count(struct shrinker *shrink, · line 7451

Quoted 3 lines
drivers/md/raid5.c +0 -0
@@ -7449,10 +7449,12 @@ static unsigned long raid5_cache_count(struct shrinker *shrink,
  {
  	struct r5conf *conf = shrink->private_data;
 

You can remove this empty line now.

drivers/md/raid5.c static unsigned long raid5_cache_count(struct shrinker *shrink, · line 7453

Quoted 3 lines
drivers/md/raid5.c +2 -1
-	if (conf->max_nr_stripes < conf->min_nr_stripes)
+	int max_stripes = READ_ONCE(conf->max_nr_stripes);
+	int min_stripes = READ_ONCE(conf->min_nr_stripes);

And please keep an emptly line between declaration. (do you check this patch with ./scripts/checkpatch.pl?)

Thanks,
Kuai

Quoted 9 lines
drivers/md/raid5.c +2 -1
+	if (max_stripes < min_stripes)
  		/* unlikely, but not impossible */
  		return 0;
-	return conf->max_nr_stripes - conf->min_nr_stripes;
+	return max_stripes - min_stripes;
  }
 
  static struct r5conf *setup_conf(struct mddev *mddev)
#2 lore raw

Hi,

Fixed in the patch v4!
Thank you for helpful advice.

Thanks,
Han