properly lock group descriptors before initializing

RFC PATCH In review Eric Sandeen 2 participants 3 messages Jul 16, 2008 ext4 file system

lore

Share & tools

lore

Eric Sandeen

author

2008-07-16 05:26 UTC #0 lore raw

RFC PATCH 1/1 properly lock group descriptors before initializing

I noticed when filling a 1T filesystem with 4 threads using the fs_mark benchmark:

fs_mark -d /mnt/test -D 256 -n 100000 -t 4 -s 20480 -F -S 0

that I occasionally got checksum mismatch errors:

EXT4-fs error (device sdb): ext4_init_inode_bitmap: Checksum bad for group 6935

etc. I'd reliably get 4-5 of them during the run.

It appears that the problem is likely a race to init the bg's when the uninit_bg feature is enabled.

With the patch below I was able to complete 2 runs with no errors or warnings. However, I did hit a hang on one run that I can't yet explain, so maybe this bears more inspection or testing.

Signed-off-by Eric Sandeen <[email protected]>

---

fs/ext4/balloc.c +3 -0

Index: linux-2.6/fs/ext4/balloc.c
===================================================================
@@ -321,12 +321,15 @@ ext4_read_block_bitmap(struct super_bloc
 	if (bh_uptodate_or_lock(bh))
 		return bh;
 
+	spin_lock(sb_bgl_lock(EXT4_SB(sb), block_group));
 	if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
 		ext4_init_block_bitmap(sb, bh, block_group, desc);
 		set_buffer_uptodate(bh);
 		unlock_buffer(bh);
+		spin_unlock(sb_bgl_lock(EXT4_SB(sb), block_group));
 		return bh;
 	}
+	spin_unlock(sb_bgl_lock(EXT4_SB(sb), block_group));
 	if (bh_submit_read(bh) < 0) {
 		put_bh(bh);
 		ext4_error(sb, __func__,

fs/ext4/ialloc.c +3 -1

Index: linux-2.6/fs/ext4/ialloc.c
===================================================================
@@ -105,6 +105,7 @@ read_inode_bitmap(struct super_block *sb
 	desc = ext4_get_group_desc(sb, block_group, NULL);
 	if (!desc)
 		goto error_out;
+	spin_lock(sb_bgl_lock(EXT4_SB(sb), block_group));
 	if (desc->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT)) {
 		bh = sb_getblk(sb, ext4_inode_bitmap(sb, desc));
 		if (!buffer_uptodate(bh)) {

@@ -119,6 +120,7 @@ read_inode_bitmap(struct super_block *sb
 	} else {
 		bh = sb_bread(sb, ext4_inode_bitmap(sb, desc));
 	}
+	spin_unlock(sb_bgl_lock(EXT4_SB(sb), block_group));
 	if (!bh)
 		ext4_error(sb, "read_inode_bitmap",
 			    "Cannot read inode bitmap - "

@@ -728,7 +730,7 @@ got:
 
 			/* When marking the block group with
 			 * ~EXT4_BG_INODE_UNINIT we don't want to depend
-			 * on the value of bg_itable_unsed even though
+			 * on the value of bg_itable_unused even though
 			 * mke2fs could have initialized the same for us.
 			 * Instead we calculated the value below
 			 */

fs/ext4/mballoc.c +3 -0

Index: linux-2.6/fs/ext4/mballoc.c
===================================================================
@@ -787,13 +787,16 @@ static int ext4_mb_init_cache(struct pag
 		if (bh_uptodate_or_lock(bh[i]))
 			continue;
 
+		spin_lock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
 		if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) {
 			ext4_init_block_bitmap(sb, bh[i],
 						first_group + i, desc);
 			set_buffer_uptodate(bh[i]);
 			unlock_buffer(bh[i]);
+			spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
 			continue;
 		}
+		spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i));
 		get_bh(bh[i]);
 		bh[i]->b_end_io = end_buffer_read_sync;
 		submit_bh(READ, bh[i]);

fs/ext4/super.c +2 -0

Index: linux-2.6/fs/ext4/super.c
===================================================================
@@ -1621,6 +1621,7 @@ static int ext4_check_descriptors(struct
 			       "(block %llu)!", i, inode_table);
 			return 0;
 		}
+		spin_lock(sb_bgl_lock(sbi, i));
 		if (!ext4_group_desc_csum_verify(sbi, i, gdp)) {
 			printk(KERN_ERR "EXT4-fs: ext4_check_descriptors: "
 			       "Checksum for group %lu failed (%u!=%u)\n",

@@ -1628,6 +1629,7 @@ static int ext4_check_descriptors(struct
 			       gdp)), le16_to_cpu(gdp->bg_checksum));
 			return 0;
 		}
+		spin_unlock(sb_bgl_lock(sbi, i));
 		if (!flexbg_flag)
 			first_block += EXT4_BLOCKS_PER_GROUP(sb);
 	}

Eric Sandeen

author

2008-07-16 05:36 UTC #1 lore raw

Quoted 9 lines · Eric Sandeen

I noticed when filling a 1T filesystem with 4 threads using the fs_mark benchmark:
fs_mark -d /mnt/test -D 256 -n 100000 -t 4 -s 20480 -F -S 0
that I occasionally got checksum mismatch errors:
EXT4-fs error (device sdb): ext4_init_inode_bitmap: Checksum bad for group 6935

Quoted 8 lines

etc. I'd reliably get 4-5 of them during the run.
It appears that the problem is likely a race to init the bg's when the uninit_bg feature is enabled.
With the patch below I was able to complete 2 runs with no errors or warnings. However, I did hit a hang on one run that I can't yet explain, so maybe this bears more inspection or testing.

Crud hit it again, looks like it's my fault. So hold off on this one :)

-Eric

Andreas Dilger

2008-07-16 09:03 UTC #2 lore raw

In reply to Eric Sandeen

fs/ext4/ialloc.c read_inode_bitmap(struct super_block *sb · line 111

Quoted 8 lines · Eric Sandeen

fs/ext4/ialloc.c +1 -0

@@ -105,6 +105,7 @@ read_inode_bitmap(struct super_block *sb
 	desc = ext4_get_group_desc(sb, block_group, NULL);
 	if (!desc)
 		goto error_out;
+	spin_lock(sb_bgl_lock(EXT4_SB(sb), block_group));
 	if (desc->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT)) {
 		bh = sb_getblk(sb, ext4_inode_bitmap(sb, desc));
 		if (!buffer_uptodate(bh)) {

sb_getblk() calls __getblk(), which is might_sleep() so is a no-no.

fs/ext4/ialloc.c read_inode_bitmap(struct super_block *sb · line 123

Quoted 5 lines

fs/ext4/ialloc.c +1 -0

@@ -119,6 +120,7 @@ read_inode_bitmap(struct super_block *sb
 	} else {
 		bh = sb_bread(sb, ext4_inode_bitmap(sb, desc));
 	}
+	spin_unlock(sb_bgl_lock(EXT4_SB(sb), block_group));

Likewise "sb_bread" is doing disk access. I guess you don't have CONFIG_DEBUG_SPINLOCK_SLEEP enabled in your kernel.

Cheers, Andreas
--
Andreas Dilger
Sr. Staff Engineer, Lustre Group
Sun Microsystems of Canada, Inc.