ext4: save the error code which triggered an ext4_error() in the superblock
Theodore Ts'o
maintainerPATCH 1/2 ext4: save the error code which triggered an ext4_error() in the superblock
This allows the cause of an ext4_error() report to be categorized based on whether it was triggered due to an I/O error, or an memory allocation error, or other possible causes. Most errors are caused by a detected file system inconsistency, so the default code stored in the superblock will be EXT4_ERR_EFSCORRUPTED.
Previous-Version-Link: https://lore.kernel.org/r/[email protected]
--- fs/ext4/balloc.c | 1 + fs/ext4/ext4.h | 30 +++++++++++++++++++- fs/ext4/ext4_jbd2.c | 3 ++ fs/ext4/extents.c | 1 + fs/ext4/ialloc.c | 2 ++ fs/ext4/inline.c | 2 ++ fs/ext4/inode.c | 8 +++++- fs/ext4/mballoc.c | 4 +++ fs/ext4/mmp.c | 6 +++- fs/ext4/namei.c | 4 +++ fs/ext4/super.c | 68 ++++++++++++++++++++++++++++++++++++++++++++- fs/ext4/xattr.c | 4 ++- 12 files changed, 128 insertions(+), 5 deletions(-)
fs/ext4/balloc.c +1 -0
@@ -506,6 +506,7 @@ int ext4_wait_block_bitmap(struct super_block *sb, ext4_group_t block_group,return -EFSCORRUPTED ;wait_on_buffer (bh);if (!buffer_uptodate (bh)) { +ext4_set_errno (sb,EIO );ext4_error (sb,"Cannot read block bitmap - " "block_group = %u, block_bitmap = %llu" , block_group, (unsigned long long ) bh->b_blocknr);
fs/ext4/ext4.h +29 -1
@@ -1343,7 +1343,8 @@ struct ext4_super_block {__u8 s_lastcheck_hi;__u8 s_first_error_time_hi;__u8 s_last_error_time_hi; -__u8 s_pad[2 ]; +__u8 s_first_error_errcode; +__u8 s_last_error_errcode;__le16 s_encoding;/* Filename charset encoding */ __le16 s_encoding_flags;/* Filename charset encoding flags */ __le32 s_reserved[95 ];/* Padding to the end of the block */
@@ -1574,6 +1575,32 @@ static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) ino <= le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)); } +/* +* Error number codes for s_{first,last}_error_errno +* +* Linux errno numbers are architecture specific, so we need to translate +* them into something which is architecture independent. We don't define +* codes for all errno's; just the ones which are most likely to be the cause +* of an ext4_error() call. +*/ +#define EXT4_ERR_UNKNOWN 1 +#define EXT4_ERR_EIO 2 +#define EXT4_ERR_ENOMEM 3 +#define EXT4_ERR_EFSBADCRC 4 +#define EXT4_ERR_EFSCORRUPTED 5 +#define EXT4_ERR_ENOSPC 6 +#define EXT4_ERR_ENOKEY 7 +#define EXT4_ERR_EROFS 8 +#define EXT4_ERR_EFBIG 9 +#define EXT4_ERR_EEXIST 10 +#define EXT4_ERR_ERANGE 11 +#define EXT4_ERR_EOVERFLOW 12 +#define EXT4_ERR_EBUSY 13 +#define EXT4_ERR_ENOTDIR 14 +#define EXT4_ERR_ENOTEMPTY 15 +#define EXT4_ERR_ESHUTDOWN 16 +#define EXT4_ERR_EFAULT 17 +/* * Inode dynamic state flags */
@@ -2686,6 +2713,7 @@ extern const char *ext4_decode_error(struct super_block *sb, int errno,extern void ext4_mark_group_bitmap_corrupted (struct super_block *sb,ext4_group_t block_group,unsigned int flags); +extern void ext4_set_errno (struct super_block *sb,int err); extern __printf(4, 5) void __ext4_error(struct super_block *, const char *, unsigned int,
fs/ext4/ext4_jbd2.c +3 -0
@@ -58,6 +58,7 @@ static int ext4_journal_check_start(struct super_block *sb)* take the FS itself readonly cleanly. */ if (journal &&is_journal_aborted (journal)) { +ext4_set_errno (sb, -journal->j_errno);ext4_abort (sb,"Detected aborted journal" );return -EROFS ; }
@@ -249,6 +250,7 @@ int __ext4_forget(const char *where, unsigned int line, handle_t *handle,if (err) {ext4_journal_abort_handle (where, line, __func__, bh, handle, err); +ext4_set_errno (inode->i_sb, -err);__ext4_abort (inode->i_sb, where, line,"error %d when attempting revoke" , err); }
@@ -320,6 +322,7 @@ int __ext4_handle_dirty_metadata(const char *where, unsigned int line, es =EXT4_SB (inode->i_sb)->s_es; es->s_last_error_block =cpu_to_le64 (bh->b_blocknr); +ext4_set_errno (inode->i_sb,EIO );ext4_error_inode (inode, where, line, bh->b_blocknr,"IO error syncing itable block" );
fs/ext4/extents.c +1 -0
@@ -492,6 +492,7 @@ static int __ext4_ext_check(const char *function, unsigned int line,return 0 ; corrupted: +ext4_set_errno (inode->i_sb, -err); ext4_error_inode(inode, function, line, 0, "pblk %llu bad header/extent: %s - magic %x, " "entries %u, max %u(%u), depth %u(%u)",
fs/ext4/ialloc.c +2 -0
@@ -194,6 +194,7 @@ ext4_read_inode_bitmap(struct super_block *sb, ext4_group_t block_group)wait_on_buffer (bh);if (!buffer_uptodate (bh)) {put_bh (bh); +ext4_set_errno (sb,EIO );ext4_error (sb,"Cannot read inode bitmap - " "block_group = %u, inode_bitmap = %llu" , block_group, bitmap_blk);
@@ -1228,6 +1229,7 @@ struct inode *ext4_orphan_get(struct super_block *sb, unsigned long ino) inode =ext4_iget (sb, ino,EXT4_IGET_NORMAL );if (IS_ERR (inode)) { err =PTR_ERR (inode); +ext4_set_errno (sb, -err);ext4_error (sb,"couldn't read orphan inode %lu (err %d)" , ino, err);return inode;
fs/ext4/inline.c +2 -0
@@ -98,6 +98,7 @@ int ext4_get_max_inline_size(struct inode *inode) error =ext4_get_inode_loc (inode, &iloc);if (error) { +ext4_set_errno (inode->i_sb, -error);ext4_error_inode (inode, __func__, __LINE__,0 ,"can't get inode location %lu" , inode->i_ino);
@@ -1761,6 +1762,7 @@ bool empty_inline_dir(struct inode *dir, int *has_inline_data) err =ext4_get_inode_loc (dir, &iloc);if (err) { +ext4_set_errno (dir->i_sb, -err);EXT4_ERROR_INODE (dir,"error %d getting inode %lu block" , err, dir->i_ino);return true;
fs/ext4/inode.c +7 -1
@@ -271,6 +271,7 @@ void ext4_evict_inode(struct inode *inode)if (inode->i_blocks) { err =ext4_truncate (inode);if (err) { +ext4_set_errno (inode->i_sb, -err);ext4_error (inode->i_sb,"couldn't truncate inode %lu (err %d)" , inode->i_ino, err);
@@ -2478,10 +2479,12 @@ static int mpage_map_and_submit_extent(handle_t *handle,EXT4_I (inode)->i_disksize = disksize;up_write (&EXT4_I (inode)->i_data_sem); err2 =ext4_mark_inode_dirty (handle, inode); -if (err2) +if (err2) { +ext4_set_errno (inode->i_sb, -err2);ext4_error (inode->i_sb,"Failed to mark inode %lu dirty" , inode->i_ino); + }if (!err) err = err2; }
@@ -4338,6 +4341,7 @@ static int __ext4_get_inode_loc(struct inode *inode,blk_finish_plug (&plug);wait_on_buffer (bh);if (!buffer_uptodate (bh)) { +ext4_set_errno (inode->i_sb,EIO );EXT4_ERROR_INODE_BLOCK (inode, block,"unable to read itable block" );brelse (bh);
@@ -4552,6 +4556,7 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, }if (!ext4_inode_csum_verify (inode, raw_inode, ei)) { +ext4_set_errno (inode->i_sb,EFSBADCRC );ext4_error_inode (inode, function, line,0 ,"iget: checksum invalid" ); ret = -EFSBADCRC ;
@@ -5090,6 +5095,7 @@ int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)if (wbc->sync_mode ==WB_SYNC_ALL && !wbc->for_sync)sync_dirty_buffer (iloc.bh);if (buffer_req (iloc.bh) && !buffer_uptodate (iloc.bh)) { +ext4_set_errno (inode->i_sb,EIO );EXT4_ERROR_INODE_BLOCK (inode, iloc.bh->b_blocknr,"IO error syncing inode" ); err = -EIO ;
fs/ext4/mballoc.c +4 -0
@@ -3895,6 +3895,7 @@ ext4_mb_discard_group_preallocations(struct super_block *sb, bitmap_bh =ext4_read_block_bitmap (sb, group);if (IS_ERR (bitmap_bh)) { err =PTR_ERR (bitmap_bh); +ext4_set_errno (sb, -err);ext4_error (sb,"Error %d reading block bitmap for %u" , err, group);return 0 ;
@@ -4063,6 +4064,7 @@ void ext4_discard_preallocations(struct inode *inode) err =ext4_mb_load_buddy_gfp (sb, group, &e4b,GFP_NOFS |__GFP_NOFAIL);if (err) { +ext4_set_errno (sb, -err);ext4_error (sb,"Error %d loading buddy information for %u" , err, group);continue ;
@@ -4071,6 +4073,7 @@ void ext4_discard_preallocations(struct inode *inode) bitmap_bh =ext4_read_block_bitmap (sb, group);if (IS_ERR (bitmap_bh)) { err =PTR_ERR (bitmap_bh); +ext4_set_errno (sb, -err);ext4_error (sb,"Error %d reading block bitmap for %u" , err, group);ext4_mb_unload_buddy (&e4b);
@@ -4325,6 +4328,7 @@ ext4_mb_discard_lg_preallocations(struct super_block *sb, err =ext4_mb_load_buddy_gfp (sb, group, &e4b,GFP_NOFS |__GFP_NOFAIL);if (err) { +ext4_set_errno (sb, -err);ext4_error (sb,"Error %d loading buddy information for %u" , err, group);continue ;
fs/ext4/mmp.c +5 -1
@@ -173,8 +173,10 @@ static int kmmpd(void *data)* (s_mmp_update_interval * 60) seconds. */ if (retval) { -if ((failed_writes %60 ) ==0 ) +if ((failed_writes %60 ) ==0 ) { +ext4_set_errno (sb, -retval);ext4_error (sb,"Error writing to MMP block" ); + } failed_writes++; }
@@ -205,6 +207,7 @@ static int kmmpd(void *data) retval =read_mmp_block (sb, &bh_check, mmp_block);if (retval) { +ext4_set_errno (sb, -retval);ext4_error (sb,"error reading MMP data: %d" , retval); goto exit_thread;
@@ -218,6 +221,7 @@ static int kmmpd(void *data)"Error while updating MMP info. " "The filesystem seems to have been" " multiply mounted."); +ext4_set_errno (sb,EBUSY );ext4_error (sb,"abort" );put_bh (bh_check); retval = -EBUSY ;
fs/ext4/namei.c +4 -0
@@ -156,6 +156,7 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode,if (ext4_dx_csum_verify (inode, dirent))set_buffer_verified (bh);else { +ext4_set_errno (inode->i_sb,EFSBADCRC );ext4_error_inode (inode, func, line, block,"Directory index failed checksum" );brelse (bh);
@@ -166,6 +167,7 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode,if (ext4_dirblock_csum_verify (inode, bh))set_buffer_verified (bh);else { +ext4_set_errno (inode->i_sb,EFSBADCRC );ext4_error_inode (inode, func, line, block,"Directory block failed checksum" );brelse (bh);
@@ -1527,6 +1529,7 @@ static struct buffer_head *__ext4_find_entry(struct inode *dir, goto next;wait_on_buffer (bh);if (!buffer_uptodate (bh)) { +ext4_set_errno (sb,EIO );EXT4_ERROR_INODE (dir,"reading directory lblock %lu" , (unsigned long ) block);brelse (bh);
@@ -1537,6 +1540,7 @@ static struct buffer_head *__ext4_find_entry(struct inode *dir, !is_dx_internal_node(dir, block, (struct ext4_dir_entry *)bh->b_data) && !ext4_dirblock_csum_verify(dir, bh)) { +ext4_set_errno (sb,EFSBADCRC );EXT4_ERROR_INODE (dir,"checksumming directory " "block %lu" , (unsigned long )block);brelse (bh);
fs/ext4/super.c +67 -1
99 lines not on this page · raw message
fs/ext4/xattr.c +3 -1
@@ -2879,9 +2879,11 @@ int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, bh =ext4_sb_bread (inode->i_sb,EXT4_I (inode)->i_file_acl,REQ_PRIO );if (IS_ERR (bh)) { error =PTR_ERR (bh); -if (error == -EIO ) +if (error == -EIO ) { +ext4_set_errno (inode->i_sb,EIO );EXT4_ERROR_INODE (inode,"block %llu read error" ,EXT4_I (inode)->i_file_acl); + } bh =NULL ; goto cleanup; }
Signature · 1 line
2.23.0
Theodore Ts'o
maintainerPATCH 2/2 ext4: simulate various I/O and checksum errors when reading metadata
This allows us to test various error handling code paths
--- fs/ext4/balloc.c | 4 +++- fs/ext4/ext4.h | 42 ++++++++++++++++++++++++++++++++++++++++++ fs/ext4/ialloc.c | 4 +++- fs/ext4/inode.c | 6 +++++- fs/ext4/namei.c | 11 ++++++++--- fs/ext4/sysfs.c | 23 +++++++++++++++++++++++ 6 files changed, 84 insertions(+), 6 deletions(-)
fs/ext4/balloc.c +3 -1
@@ -371,7 +371,8 @@ static int ext4_validate_block_bitmap(struct super_block *sb,if (buffer_verified (bh)) goto verified;if (unlikely (!ext4_block_bitmap_csum_verify (sb, block_group, - desc, bh))) { + desc, bh) || +ext4_simulate_fail (sb,EXT4_SIM_BBITMAP_CRC ))) {ext4_unlock_group (sb, block_group);ext4_error (sb,"bg %u: bad block bitmap checksum" , block_group); ext4_mark_group_bitmap_corrupted(sb, block_group,
@@ -505,6 +506,7 @@ int ext4_wait_block_bitmap(struct super_block *sb, ext4_group_t block_group,if (!desc)return -EFSCORRUPTED ;wait_on_buffer (bh); +ext4_simulate_fail_bh (sb, bh,EXT4_SIM_BBITMAP_EIO );if (!buffer_uptodate (bh)) {ext4_set_errno (sb,EIO ); ext4_error(sb, "Cannot read block bitmap - "
fs/ext4/ext4.h +42 -0
54 lines not on this page · raw message
fs/ext4/ialloc.c +3 -1
@@ -94,7 +94,8 @@ static int ext4_validate_inode_bitmap(struct super_block *sb, goto verified; blk =ext4_inode_bitmap (sb, desc);if (!ext4_inode_bitmap_csum_verify (sb, block_group, desc, bh, -EXT4_INODES_PER_GROUP (sb) /8 )) { +EXT4_INODES_PER_GROUP (sb) /8 ) || +ext4_simulate_fail (sb,EXT4_SIM_IBITMAP_CRC )) {ext4_unlock_group (sb, block_group);ext4_error (sb,"Corrupt inode bitmap - block_group = %u, " "inode_bitmap = %llu" , block_group, blk);
@@ -192,6 +193,7 @@ ext4_read_inode_bitmap(struct super_block *sb, ext4_group_t block_group)get_bh (bh);submit_bh (REQ_OP_READ ,REQ_META |REQ_PRIO , bh);wait_on_buffer (bh); +ext4_simulate_fail_bh (sb, bh,EXT4_SIM_IBITMAP_EIO );if (!buffer_uptodate (bh)) {put_bh (bh);ext4_set_errno (sb,EIO );
fs/ext4/inode.c +5 -1
@@ -4243,6 +4243,8 @@ static int __ext4_get_inode_loc(struct inode *inode, bh =sb_getblk (sb, block);if (unlikely (!bh))return -ENOMEM ; +if (ext4_simulate_fail (sb,EXT4_SIM_INODE_EIO )) + goto simulate_eio;if (!buffer_uptodate (bh)) {lock_buffer (bh);
@@ -4341,6 +4343,7 @@ static int __ext4_get_inode_loc(struct inode *inode,blk_finish_plug (&plug);wait_on_buffer (bh);if (!buffer_uptodate (bh)) { + simulate_eio:ext4_set_errno (inode->i_sb,EIO );EXT4_ERROR_INODE_BLOCK (inode, block,"unable to read itable block" );
@@ -4555,7 +4558,8 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, sizeof(gen)); } -if (!ext4_inode_csum_verify (inode, raw_inode, ei)) { +if (!ext4_inode_csum_verify (inode, raw_inode, ei) || +ext4_simulate_fail (sb,EXT4_SIM_INODE_CRC )) {ext4_set_errno (inode->i_sb,EFSBADCRC );ext4_error_inode (inode, function, line,0 ,"iget: checksum invalid" );
fs/ext4/namei.c +8 -3
@@ -109,7 +109,10 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode,struct ext4_dir_entry *dirent;int is_dx_block =0 ; - bh =ext4_bread (NULL , inode, block,0 ); +if (ext4_simulate_fail (inode->i_sb,EXT4_SIM_DIRBLOCK_EIO )) + bh =ERR_PTR (-EIO ); +else + bh =ext4_bread (NULL , inode, block,0 );if (IS_ERR (bh)) {__ext4_warning (inode->i_sb, func, line, "inode #%lu: lblock %lu: comm %s: "
@@ -153,7 +156,8 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode,* caller is sure it should be an index block. */ if (is_dx_block && type ==INDEX ) { -if (ext4_dx_csum_verify (inode, dirent)) +if (ext4_dx_csum_verify (inode, dirent) && + !ext4_simulate_fail (inode->i_sb,EXT4_SIM_DIRBLOCK_CRC ))set_buffer_verified (bh);else {ext4_set_errno (inode->i_sb,EFSBADCRC );
@@ -164,7 +168,8 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode, } }if (!is_dx_block) { -if (ext4_dirblock_csum_verify (inode, bh)) +if (ext4_dirblock_csum_verify (inode, bh) && + !ext4_simulate_fail (inode->i_sb,EXT4_SIM_DIRBLOCK_CRC ))set_buffer_verified (bh);else {ext4_set_errno (inode->i_sb,EFSBADCRC );
fs/ext4/sysfs.c +23 -0
@@ -29,6 +29,7 @@ typedef enum { attr_last_error_time, attr_feature, attr_pointer_ui, + attr_pointer_ul, attr_pointer_atomic, attr_journal_task, } attr_id_t;
@@ -160,6 +161,9 @@ static struct ext4_attr ext4_attr_##_name = { \#define EXT4_RW_ATTR_SBI_UI (_name,_elname) \ EXT4_ATTR_OFFSET(_name, 0644, pointer_ui, ext4_sb_info, _elname) +#define EXT4_RW_ATTR_SBI_UL (_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0644, pointer_ul, ext4_sb_info, _elname) +#define EXT4_ATTR_PTR (_name,_mode,_id,_ptr) \ static struct ext4_attr ext4_attr_##_name = { \ .attr = {.name = __stringify(_name), .mode = _mode }, \
@@ -194,6 +198,9 @@ EXT4_RW_ATTR_SBI_UI(warning_ratelimit_interval_ms, s_warning_ratelimit_state.intEXT4_RW_ATTR_SBI_UI (warning_ratelimit_burst, s_warning_ratelimit_state.burst);EXT4_RW_ATTR_SBI_UI (msg_ratelimit_interval_ms, s_msg_ratelimit_state.interval);EXT4_RW_ATTR_SBI_UI (msg_ratelimit_burst, s_msg_ratelimit_state.burst); +#ifdef CONFIG_EXT4_DEBUG +EXT4_RW_ATTR_SBI_UL (simulate_fail, s_simulate_fail); +#endif EXT4_RO_ATTR_ES_UI (errors_count, s_error_count);EXT4_ATTR (first_error_time,0444 , first_error_time);EXT4_ATTR (last_error_time,0444 , last_error_time);
@@ -228,6 +235,9 @@ static struct attribute *ext4_attrs[] = {ATTR_LIST (first_error_time),ATTR_LIST (last_error_time),ATTR_LIST (journal_task), +#ifdef CONFIG_EXT4_DEBUG +ATTR_LIST (simulate_fail), +#endifNULL , };ATTRIBUTE_GROUPS (ext4);
@@ -318,6 +328,11 @@ static ssize_t ext4_attr_show(struct kobject *kobj,else return snprintf (buf,PAGE_SIZE ,"%u\n" , *((unsigned int *) ptr)); +case attr_pointer_ul: +if (!ptr) +return 0 ; +return snprintf (buf,PAGE_SIZE ,"%lu\n" , + *((unsigned long *) ptr));case attr_pointer_atomic:if (!ptr)return 0 ;
@@ -361,6 +376,14 @@ static ssize_t ext4_attr_store(struct kobject *kobj,else *((unsigned int *) ptr) = t;return len; +case attr_pointer_ul: +if (!ptr) +return 0 ; + ret =kstrtoul (skip_spaces (buf),0 , &t); +if (ret) +return ret; + *((unsigned long *) ptr) = t; +return len;case attr_inode_readahead:return inode_readahead_blks_store (sbi, buf, len);case attr_trigger_test_error:
Signature · 1 line
2.23.0
Re: [PATCH 2/2] ext4: simulate various I/O and checksum errors when reading metadata
Quoted 49 lines · Theodore Ts'o
This allows us to test various error handling code paths
Signed-off-by Theodore Ts'o <[email protected]>--- fs/ext4/balloc.c | 4 +++- fs/ext4/ext4.h | 42 ++++++++++++++++++++++++++++++++++++++++++ fs/ext4/ialloc.c | 4 +++- fs/ext4/inode.c | 6 +++++- fs/ext4/namei.c | 11 ++++++++--- fs/ext4/sysfs.c | 23 +++++++++++++++++++++++ 6 files changed, 84 insertions(+), 6 deletions(-)fs/ext4/balloc.c +3
-1@@ -371,7 +371,8 @@ static int ext4_validate_block_bitmap(struct super_block *sb,if (buffer_verified (bh)) goto verified;if (unlikely (!ext4_block_bitmap_csum_verify (sb, block_group, - desc, bh))) { + desc, bh) || +ext4_simulate_fail (sb,EXT4_SIM_BBITMAP_CRC ))) {ext4_unlock_group (sb, block_group);ext4_error (sb,"bg %u: bad block bitmap checksum" , block_group); ext4_mark_group_bitmap_corrupted(sb, block_group,@@ -505,6 +506,7 @@ int ext4_wait_block_bitmap(struct super_block *sb, ext4_group_t block_group,if (!desc)return -EFSCORRUPTED ;wait_on_buffer (bh); +ext4_simulate_fail_bh (sb, bh,EXT4_SIM_BBITMAP_EIO );if (!buffer_uptodate (bh)) {ext4_set_errno (sb,EIO ); ext4_error(sb, "Cannot read block bitmap - "fs/ext4/ext4.h +3
-0@@ -1557,6 +1557,9 @@ struct ext4_sb_info {/* Barrier between changing inodes' journal flags and writepages ops. */ struct percpu_rw_semaphore s_journal_flag_rwsem;struct dax_device *s_daxdev; +#ifdef CONFIG_EXT4_DEBUG +unsigned long s_simulate_fail; +#endif };static inline struct ext4_sb_info *EXT4_SB (struct super_block *sb)
fs/ext4/ext4.h static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) · line 1582
Quoted 5 lines
fs/ext4/ext4.h +2
-0@@ -1575,6 +1578,45 @@ static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) ino <= le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)); } +static inline int ext4_simulate_fail (struct super_block *sb, +unsigned long flag)
Nit: bool?
fs/ext4/ext4.h static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) · line 1593
Quoted 11 lines
fs/ext4/ext4.h +11
-0+{ +#ifdef CONFIG_EXT4_DEBUG +unsigned long old, new; +struct ext4_sb_info *sbi =EXT4_SB (sb); + +do { + old =READ_ONCE (sbi->s_simulate_fail); +if (likely ((old & flag) ==0 )) +return 0 ; + new = old & ~flag; + }while (unlikely (cmpxchg (&sbi->s_simulate_fail, old, new) != old));
If I'm reading this correctly, this means that userspace sets a s_simulate_fail bit via sysfs knob, and the next time the filesystem calls ext4_simulate_fail with the same bit set in @flag we'll return true to say "simulate the failure" and clear the bit in s_simulate_fail?
IOWs, the simulated failures have to be re-armed every time?
Seems reasonable, but consider the possibility that in the future it might be useful if you could set up periodic failures (e.g. directory lookups fail 10% of the time) so that you can see how something like fsstress reacts to less-predictable failures?
Of course that also increases the amount of fugly sysfs boilerplate so that each knob can have its own sysfs file... that alone is half of a reason not to do that. :(
--D
Quoted 190 lines
fs/ext4/ext4.h +26
-0+return 1 ; +#else +return 0 ; +#endif +} + +static inline void ext4_simulate_fail_bh (struct super_block *sb, +struct buffer_head *bh, +unsigned long flag) +{ +if (!IS_ERR (bh) &&ext4_simulate_fail (sb, flag)) +clear_buffer_uptodate (bh); +} + +/* +* Simulate_fail flags +*/ +#define EXT4_SIM_BBITMAP_EIO 0x00000001 +#define EXT4_SIM_BBITMAP_CRC 0x00000002 +#define EXT4_SIM_IBITMAP_EIO 0x00000004 +#define EXT4_SIM_IBITMAP_CRC 0x00000008 +#define EXT4_SIM_INODE_EIO 0x00000010 +#define EXT4_SIM_INODE_CRC 0x00000020 +#define EXT4_SIM_DIRBLOCK_EIO 0x00000040 +#define EXT4_SIM_DIRBLOCK_CRC 0x00000080 +/* * Error number codes for s_{first,last}_error_errno * fs/ext4/ialloc.c +3
-1@@ -94,7 +94,8 @@ static int ext4_validate_inode_bitmap(struct super_block *sb, goto verified; blk =ext4_inode_bitmap (sb, desc);if (!ext4_inode_bitmap_csum_verify (sb, block_group, desc, bh, -EXT4_INODES_PER_GROUP (sb) /8 )) { +EXT4_INODES_PER_GROUP (sb) /8 ) || +ext4_simulate_fail (sb,EXT4_SIM_IBITMAP_CRC )) {ext4_unlock_group (sb, block_group);ext4_error (sb,"Corrupt inode bitmap - block_group = %u, " "inode_bitmap = %llu" , block_group, blk);@@ -192,6 +193,7 @@ ext4_read_inode_bitmap(struct super_block *sb, ext4_group_t block_group)get_bh (bh);submit_bh (REQ_OP_READ ,REQ_META |REQ_PRIO , bh);wait_on_buffer (bh); +ext4_simulate_fail_bh (sb, bh,EXT4_SIM_IBITMAP_EIO );if (!buffer_uptodate (bh)) {put_bh (bh);ext4_set_errno (sb,EIO );fs/ext4/inode.c +5
-1@@ -4243,6 +4243,8 @@ static int __ext4_get_inode_loc(struct inode *inode, bh =sb_getblk (sb, block);if (unlikely (!bh))return -ENOMEM ; +if (ext4_simulate_fail (sb,EXT4_SIM_INODE_EIO )) + goto simulate_eio;if (!buffer_uptodate (bh)) {lock_buffer (bh);@@ -4341,6 +4343,7 @@ static int __ext4_get_inode_loc(struct inode *inode,blk_finish_plug (&plug);wait_on_buffer (bh);if (!buffer_uptodate (bh)) { + simulate_eio:ext4_set_errno (inode->i_sb,EIO );EXT4_ERROR_INODE_BLOCK (inode, block,"unable to read itable block" );@@ -4555,7 +4558,8 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, sizeof(gen)); } -if (!ext4_inode_csum_verify (inode, raw_inode, ei)) { +if (!ext4_inode_csum_verify (inode, raw_inode, ei) || +ext4_simulate_fail (sb,EXT4_SIM_INODE_CRC )) {ext4_set_errno (inode->i_sb,EFSBADCRC );ext4_error_inode (inode, function, line,0 ,"iget: checksum invalid" );fs/ext4/namei.c +8
-3@@ -109,7 +109,10 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode,struct ext4_dir_entry *dirent;int is_dx_block =0 ; - bh =ext4_bread (NULL , inode, block,0 ); +if (ext4_simulate_fail (inode->i_sb,EXT4_SIM_DIRBLOCK_EIO )) + bh =ERR_PTR (-EIO ); +else + bh =ext4_bread (NULL , inode, block,0 );if (IS_ERR (bh)) {__ext4_warning (inode->i_sb, func, line, "inode #%lu: lblock %lu: comm %s: "@@ -153,7 +156,8 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode,* caller is sure it should be an index block. */ if (is_dx_block && type ==INDEX ) { -if (ext4_dx_csum_verify (inode, dirent)) +if (ext4_dx_csum_verify (inode, dirent) && + !ext4_simulate_fail (inode->i_sb,EXT4_SIM_DIRBLOCK_CRC ))set_buffer_verified (bh);else {ext4_set_errno (inode->i_sb,EFSBADCRC );@@ -164,7 +168,8 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode, } }if (!is_dx_block) { -if (ext4_dirblock_csum_verify (inode, bh)) +if (ext4_dirblock_csum_verify (inode, bh) && + !ext4_simulate_fail (inode->i_sb,EXT4_SIM_DIRBLOCK_CRC ))set_buffer_verified (bh);else {ext4_set_errno (inode->i_sb,EFSBADCRC );fs/ext4/sysfs.c +23
-0@@ -29,6 +29,7 @@ typedef enum { attr_last_error_time, attr_feature, attr_pointer_ui, + attr_pointer_ul, attr_pointer_atomic, attr_journal_task, } attr_id_t;@@ -160,6 +161,9 @@ static struct ext4_attr ext4_attr_##_name = { \#define EXT4_RW_ATTR_SBI_UI (_name,_elname) \ EXT4_ATTR_OFFSET(_name, 0644, pointer_ui, ext4_sb_info, _elname) +#define EXT4_RW_ATTR_SBI_UL (_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0644, pointer_ul, ext4_sb_info, _elname) +#define EXT4_ATTR_PTR (_name,_mode,_id,_ptr) \ static struct ext4_attr ext4_attr_##_name = { \ .attr = {.name = __stringify(_name), .mode = _mode }, \@@ -194,6 +198,9 @@ EXT4_RW_ATTR_SBI_UI(warning_ratelimit_interval_ms, s_warning_ratelimit_state.intEXT4_RW_ATTR_SBI_UI (warning_ratelimit_burst, s_warning_ratelimit_state.burst);EXT4_RW_ATTR_SBI_UI (msg_ratelimit_interval_ms, s_msg_ratelimit_state.interval);EXT4_RW_ATTR_SBI_UI (msg_ratelimit_burst, s_msg_ratelimit_state.burst); +#ifdef CONFIG_EXT4_DEBUG +EXT4_RW_ATTR_SBI_UL (simulate_fail, s_simulate_fail); +#endif EXT4_RO_ATTR_ES_UI (errors_count, s_error_count);EXT4_ATTR (first_error_time,0444 , first_error_time);EXT4_ATTR (last_error_time,0444 , last_error_time);@@ -228,6 +235,9 @@ static struct attribute *ext4_attrs[] = {ATTR_LIST (first_error_time),ATTR_LIST (last_error_time),ATTR_LIST (journal_task), +#ifdef CONFIG_EXT4_DEBUG +ATTR_LIST (simulate_fail), +#endifNULL , };ATTRIBUTE_GROUPS (ext4);@@ -318,6 +328,11 @@ static ssize_t ext4_attr_show(struct kobject *kobj,else return snprintf (buf,PAGE_SIZE ,"%u\n" , *((unsigned int *) ptr)); +case attr_pointer_ul: +if (!ptr) +return 0 ; +return snprintf (buf,PAGE_SIZE ,"%lu\n" , + *((unsigned long *) ptr));case attr_pointer_atomic:if (!ptr)return 0 ;@@ -361,6 +376,14 @@ static ssize_t ext4_attr_store(struct kobject *kobj,else *((unsigned int *) ptr) = t;return len; +case attr_pointer_ul: +if (!ptr) +return 0 ; + ret =kstrtoul (skip_spaces (buf),0 , &t); +if (ret) +return ret; + *((unsigned long *) ptr) = t; +return len;case attr_inode_readahead:return inode_readahead_blks_store (sbi, buf, len);case attr_trigger_test_error:Signature · 1 line
2.23.0
Theodore Y. Ts'o
maintainerRe: [PATCH 2/2] ext4: simulate various I/O and checksum errors when reading metadata
Quoted 4 lines · Darrick J. Wong
Review
Quoted 2 lines
(quoted diff) +2
-0+static inline int ext4_simulate_fail(struct super_block *sb, + unsigned long flag)Nit: bool?
Sure, I'll do this for the next version.
Quoted 6 lines
If I'm reading this correctly, this means that userspace sets a
s_simulate_failbit via sysfs knob, and the next time the filesystem callsext4_simulate_failwith the same bit set in @flag we'll return true to say "simulate the failure" and clear the bit ins_simulate_fail?IOWs, the simulated failures have to be re-armed every time?
Yes, that's correct.
Quoted 4 lines
Seems reasonable, but consider the possibility that in the future it might be useful if you could set up periodic failures (e.g. directory lookups fail 10% of the time) so that you can see how something like fsstress reacts to less-predictable failures?
So in theory, we could do that with dm_flakey --- but that's a pain in the tuckus, since you have to specify the LBA for the directory blocks that you might want to have fail. I implemented this so I could have a quick and dirty way of testing the first patch in this series (and in fact, I found a bug in the first version of the previous patch, so I'm glad I spent the time to implement the test patch :-).
What might be interesting to do is some kind of eBPF hook where we pass in the block #, inode #, and metadata type, and the ePBF program could do use a much more complex set of criteria in terms of whether or not to trigger an EIO, or how to fuzz a particular block to either force a CRC failure, or to try to find bugs ala Hydra[1] (funded via a Google Faculty Research Award grant), but using a much more glass-box style test approach.
[1] https://gts3.org/~sanidhya/pubs/2019/hydra.pdf
This would be a lot more work, and I'm not sufficiently up to speed with eBPF, and I just needed a quick and dirty testing scheme.
The reason why I think it's worthwhile to land this patch (as opposed to throwing it away after doing the development work for the previous patch) is that it's a relatively small set of changes, and all of the code disappears if CONFIG_DEBUG_EXT4 is not enabled. So it has no performance cost on production kernels, and it's highly unlikely that users would have a reason to use this feature on production use cases, so ripping this out if and when we have a more functional eBPF testing infrastructure to replace it shouldn't really be a problem.
- Ted
P.S. A fascinating question is whether we could make the hooks for this hypothetical eBPF hook general enough that it could work for more than just ext4, but for other file systems. The problem is that the fs metadata types are not going to be same across different file systems, so that makes the API design quite tricky; and perhaps not worth it?
Re: [PATCH 2/2] ext4: simulate various I/O and checksum errors when reading metadata
Quoted 17 lines · Theodore Y. Ts'o
Quoted 4 lines · Darrick J. Wong
Review
Quoted 2 lines
(quoted diff) +2
-0+static inline int ext4_simulate_fail(struct super_block *sb, + unsigned long flag)Nit: bool?
Sure, I'll do this for the next version.
Quoted 6 lines
If I'm reading this correctly, this means that userspace sets a
s_simulate_failbit via sysfs knob, and the next time the filesystem callsext4_simulate_failwith the same bit set in @flag we'll return true to say "simulate the failure" and clear the bit ins_simulate_fail?IOWs, the simulated failures have to be re-armed every time?
Yes, that's correct.
Quoted 8 lines
Quoted 4 lines
Seems reasonable, but consider the possibility that in the future it might be useful if you could set up periodic failures (e.g. directory lookups fail 10% of the time) so that you can see how something like fsstress reacts to less-predictable failures?
So in theory, we could do that with
dm_flakey--- but that's a pain in the tuckus, since you have to specify the LBA for the directory blocks that you might want to have fail.
Funny, I've been working on a fstests helper function to make it easy to set up dm-flakey based on fiemap/getfsmap output and such. :)
Quoted 4 lines
I implemented this so I could have
a quick and dirty way of testing the first patch in this series (and in fact, I found a bug in the first version of the previous patch, so I'm glad I spent the time to implement the test patch :-).
Heh, cool!
Quoted 7 lines
What might be interesting to do is some kind of eBPF hook where we pass in the block #, inode #, and metadata type, and the ePBF program could do use a much more complex set of criteria in terms of whether or not to trigger an EIO, or how to fuzz a particular block to either force a CRC failure, or to try to find bugs ala Hydra[1] (funded via a Google Faculty Research Award grant), but using a much more glass-box style test approach.
That would be fun. Attach an arbitrary eBPF program to a range of sectors. I wonder how loud the howls of protest would be for "can we let ebpf programs scribble on a kernel io buffer pleeze?"...
...a couple of years ago I sent out an RFCRAP patch so that you could use eBPF's "new" ability to change function return values, which Christoph immediately NAKd. I think Josef's original purpose was so that he could inject arbitrary debugging knobs all over btrfs.
Quoted 5 lines
[1] https://gts3.org/~sanidhya/pubs/2019/hydra.pdf
This would be a lot more work, and I'm not sufficiently up to speed with eBPF, and I just needed a quick and dirty testing scheme.
Quoted 8 lines
The reason why I think it's worthwhile to land this patch (as opposed to throwing it away after doing the development work for the previous patch) is that it's a relatively small set of changes, and all of the code disappears if
CONFIG_DEBUG_EXT4is not enabled. So it has no performance cost on production kernels, and it's highly unlikely that users would have a reason to use this feature on production use cases, so ripping this out if and when we have a more functional eBPF testing infrastructure to replace it shouldn't really be a problem.
Admittedly it's a debug knob so I don't see it as a big deal if you merge this and some day rip it out or supersede it. The XFS knobs have undergone a few, uh, interface revisions.
Quoted 8 lines
- Ted
P.S. A fascinating question is whether we could make the hooks for this hypothetical eBPF hook general enough that it could work for more than just ext4, but for other file systems. The problem is that the fs metadata types are not going to be same across different file systems, so that makes the API design quite tricky; and perhaps not worth it?
Yeah. I mean, it's eBPF glomming onto random parts of the kernel, so I don't think there's ever going to be a General API For Brain Slugs[3].
OTOH I need LSF topics so sure lets roll.
--D
[2] https://lore.kernel.org/linux-xfs/20171213061825.GO19219@magnolia/
[3] https://futurama.fandom.com/wiki/Brain_Slug
Theodore Y. Ts'o
maintainerRe: [PATCH 2/2] ext4: simulate various I/O and checksum errors when reading metadata
Quoted 6 lines · Darrick J. Wong
Quoted 3 lines
So in theory, we could do that with
dm_flakey--- but that's a pain in the tuckus, since you have to specify the LBA for the directory blocks that you might want to have fail.Funny, I've been working on a fstests helper function to make it easy to set up dm-flakey based on fiemap/getfsmap output and such. :)
Ah, excellent, I'm looking forward to seeing it. :-)
Quoted 11 lines
Quoted 7 lines
What might be interesting to do is some kind of eBPF hook where we pass in the block #, inode #, and metadata type, and the ePBF program could do use a much more complex set of criteria in terms of whether or not to trigger an EIO, or how to fuzz a particular block to either force a CRC failure, or to try to find bugs ala Hydra[1] (funded via a Google Faculty Research Award grant), but using a much more glass-box style test approach.
That would be fun. Attach an arbitrary eBPF program to a range of sectors. I wonder how loud the howls of protest would be for "can we let ebpf programs scribble on a kernel io buffer pleeze?"...
Well, because the eBPF hook would include the metadata type (an allocation bitmap vs inode table vs htree index vs htree leaf block, etc.) what I was thinking about has to be done in ext4 as a ext4-specific hook. And it would only be enabled if CONFIG_EXT4_DEBUG or a separate Kconfig option is enabled --- and I *hope* no distribution would be so silly/stupid enough to enable it on a product kernel build. :-)
- Ted
Re: [PATCH 2/2] ext4: simulate various I/O and checksum errors when reading metadata
In reply to Darrick J. WongQuoted 78 lines · Darrick J. Wong
Quoted 49 lines · Theodore Ts'o
This allows us to test various error handling code paths
Signed-off-by Theodore Ts'o <[email protected]>---
fs/ext4/balloc.c | 4 +++- fs/ext4/ext4.h | 42 ++++++++++++++++++++++++++++++++++++++++++ fs/ext4/ialloc.c | 4 +++- fs/ext4/inode.c | 6 +++++- fs/ext4/namei.c | 11 ++++++++--- fs/ext4/sysfs.c | 23 +++++++++++++++++++++++ 6 files changed, 84 insertions(+), 6 deletions(-)fs/ext4/balloc.c +3
-1@@ -371,7 +371,8 @@ static int ext4_validate_block_bitmap(struct super_block *sb,if (buffer_verified (bh)) goto verified;if (unlikely (!ext4_block_bitmap_csum_verify (sb, block_group, - desc, bh))) { + desc, bh) || +ext4_simulate_fail (sb,EXT4_SIM_BBITMAP_CRC ))) {ext4_unlock_group (sb, block_group);ext4_error (sb,"bg %u: bad block bitmap checksum" , block_group); ext4_mark_group_bitmap_corrupted(sb, block_group,@@ -505,6 +506,7 @@ int ext4_wait_block_bitmap(struct super_block *sb, ext4_group_t block_group,if (!desc)return -EFSCORRUPTED ;wait_on_buffer (bh); +ext4_simulate_fail_bh (sb, bh,EXT4_SIM_BBITMAP_EIO );if (!buffer_uptodate (bh)) {ext4_set_errno (sb,EIO ); ext4_error(sb, "Cannot read block bitmap - "fs/ext4/ext4.h +3
-0@@ -1557,6 +1557,9 @@ struct ext4_sb_info {/* Barrier between changing inodes' journal flags and writepages ops. */ struct percpu_rw_semaphore s_journal_flag_rwsem;struct dax_device *s_daxdev; +#ifdef CONFIG_EXT4_DEBUG +unsigned long s_simulate_fail; +#endif };static inline struct ext4_sb_info *EXT4_SB (struct super_block *sb)fs/ext4/ext4.h static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) · line 1582
Quoted 5 lines
fs/ext4/ext4.h +2
-0@@ -1575,6 +1578,45 @@ static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) ino <= le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)); } +static inline int ext4_simulate_fail (struct super_block *sb, +unsigned long flag)Nit: bool?
fs/ext4/ext4.h static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) · line 1593
Quoted 11 lines
fs/ext4/ext4.h +11
-0+{ +#ifdef CONFIG_EXT4_DEBUG +unsigned long old, new; +struct ext4_sb_info *sbi =EXT4_SB (sb); + +do { + old =READ_ONCE (sbi->s_simulate_fail); +if (likely ((old & flag) ==0 )) +return 0 ; + new = old & ~flag; + }while (unlikely (cmpxchg (&sbi->s_simulate_fail, old, new) != old));If I'm reading this correctly, this means that userspace sets a
s_simulate_failbit via sysfs knob, and the next time the filesystem callsext4_simulate_failwith the same bit set in @flag we'll return true to say "simulate the failure" and clear the bit ins_simulate_fail?IOWs, the simulated failures have to be re-armed every time?
Quoted 8 lines
Seems reasonable, but consider the possibility that in the future it might be useful if you could set up periodic failures (e.g. directory lookups fail 10% of the time) so that you can see how something like fsstress reacts to less-predictable failures?
Of course that also increases the amount of fugly sysfs boilerplate so that each knob can have its own sysfs file... that alone is half of a reason not to do that. :(
Just for comparison, Lustre has had a fault injection mechanism for ages that can do a bunch of things like this. Each fault location has a unique number (we separate them by subsystems in the code, but numbers are rather arbitrary), and then a sysfs parameter "fail_loc" that can be set to match the fault location to inject errors, and "fail_val" that allows userspace to adjust/tune the failure behavior (e.g. only affect target N, or sleep N seconds, ...).
The low 16 bits set in fail_loc is the fault location number, and the high 16 bits of fail_loc are flags that can modify the behavior independent of which failure number is being used:
- CFS_FAIL_ONCE: the fail_loc should only fail once (default forever)
- CFS_FAIL_SKIP: skip the fail_loc the first "fail_val" times
- CFS_FAIL_SOME: trigger the failure the first "fail_val" times
- CFS_FAIL_RAND: trigger the failure at a rate of 1/fail_val
There are also flags set by the kernel when the failure is hit, so it is possible to read fail_loc in a test script if the failure was already hit.
Internally in the code, the most common use is just checking if we hit the currently-set fail_loc (which is unlikely() for minimal impact), like:
if (CFS_FAIL_CHECK(OBD_FAIL_TGT_REPLAY_RECONNECT))
RETURN(1); /* don't send early reply */
if (CFS_FAIL_CHECK(OBD_FAIL_FLD_QUERY_REQ) && req->rq_no_delay) {
/* the same error returned by ptlrpc_import_delay_req() */
rc = -EWOULDBLOCK;
req->rq_status = rc;It is possible to inject a delay into a thread to allow something else to happen (maybe more useful for a distributed system rather than a local one):
CFS_FAIL_TIMEOUT(OBD_FAIL_TGT_REPLAY_DELAY2, cfs_fail_val);
It is also possible to set up a race between two threads in the same or different parts of the code on the same node:
CFS_RACE(CFS_FAIL_CHLOG_USER_REG_UNREG_RACE);
The first thread to hit this fail_loc will sleep, and the second thread that hits it will wake it up. There is a variation of this to make it explicit that only one thread to hit a location should sleep, and a second thread needs to hit a different location to wake it up:
thread1: CFS_RACE_WAIT(OBD_FAIL_OBD_ZERO_NLINK_RACE); thread2: CFS_RACE_WAKEUP(OBD_FAIL_OBD_ZERO_NLINK_RACE);
It is also possible to daisy-chain failure conditions:
if (ns_is_client(ldlm_lock_to_ns(lock)) && CFS_FAIL_CHECK_RESET(OBD_FAIL_LDLM_INTR_CP_AST, OBD_FAIL_LDLM_CP_BL_RACE | OBD_FAIL_ONCE)) ldlm_set_fail_loc(lock);
so here if "OBD_FAIL_LDLM_INTR_CP_AST" is hit, it will reset fail_loc to be "OBD_FAIL_LDLM_CP_BL_RACE" to fail once, flag a particular DLM lock and then the next two threads that access this lock will race to process it:
if (ldlm_is_fail_loc(lock)) CFS_RACE(OBD_FAIL_LDLM_CP_BL_RACE);
The CFS_FAIL functionality has been used for quite a few years and has proven sufficient and easily used to invoke failure conditions that would be otherwise impossible to reproduce (over a thousand fault injection sites in the Lustre code and corresponding tests to trigger them today).
Cheers, Andreas