fix log recovery for v5 superblocks
v2 cover fix log recovery for v5 superblocks
Hi all,
Here's v2 of the series to fix up log recovery to update metadata LSNs and handle buffer submission correctly. This is mostly similar to v1. The primary functional change is a fixup to patch 1, that came out of the discussion with Dave, to drain queued buffers on LSN changes of commit log records (as opposed to all LSN changes). Otherwise, fixes include some light refactoring and comment reworks. Thoughts, reviews, flames appreciated.
Brian
v2:
- Rebased to for-next.
- Limit LSN change trigger to log commit records.
- Comment rewrite/fixups.
- Refactor buffer validation handling of NULL current lsn callers (skipped bufs).
v1: http://oss.sgi.com/pipermail/xfs/2016-August/050840.html
Brian Foster (5):
xfs: rework log recovery to submit buffers on LSN boundaries
xfs: pass current lsn to log recovery buffer validation
xfs: don't warn on buffers not being recovered due to LSN
xfs: update metadata LSN in buffers during log recovery
xfs: log recovery tracepoints to track current lsn and buffer
submission
fs/xfs/xfs_log_priv.h | 3 +- fs/xfs/xfs_log_recover.c | 191 +++++++++++++++++++++++++++++++++-------------- fs/xfs/xfs_trace.h | 31 +++++++- 3 files changed, 166 insertions(+), 59 deletions(-)
Signature · 1 line
2.5.5
PATCH v2 1/5 xfs: rework log recovery to submit buffers on LSN boundaries
The fix to log recovery to update the metadata LSN in recovered buffers introduces the requirement that a buffer is submitted only once per current LSN. Log recovery currently submits buffers on transaction boundaries. This is not sufficient as the abstraction between log records and transactions allows for various scenarios where multiple transactions can share the same current LSN. If independent transactions share an LSN and both modify the same buffer, log recovery can incorrectly skip updates and leave the filesystem in an inconsisent state.
In preparation for proper metadata LSN updates during log recovery, update log recovery to submit buffers for write on LSN change boundaries rather than transaction boundaries. Explicitly track the current LSN in a new struct xlog field to handle the various corner cases of when the current LSN may or may not change.
--- fs/xfs/xfs_log_priv.h | 3 +- fs/xfs/xfs_log_recover.c | 82 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 66 insertions(+), 19 deletions(-)
fs/xfs/xfs_log_priv.h +2 -1
@@ -413,7 +413,8 @@ struct xlog {/* log record crc error injection factor */ uint32_t l_badcrc_factor;#endif - +/* log recovery lsn tracking (for buffer submission */ +xfs_lsn_t l_recovery_lsn; };#define XLOG_BUF_CANCEL_BUCKET (log, blkno) \
fs/xfs/xfs_log_recover.c +64 -18
193 lines not on this page · raw message
Signature · 1 line
2.5.5
PATCH v2 2/5 xfs: pass current lsn to log recovery buffer validation
In reply to Brian FosterThe current LSN must be available to the buffer validation function to provide the ability to update the metadata LSN of the buffer. Pass the current_lsn value down to xlog_recover_validate_buf_type() in preparation.
--- fs/xfs/xfs_log_recover.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-)
fs/xfs/xfs_log_recover.c +8 -6
@@ -2360,7 +2360,8 @@ static voidxlog_recover_validate_buf_type ( struct xfs_mount *mp, struct xfs_buf *bp, - xfs_buf_log_format_t *buf_f) + xfs_buf_log_format_t *buf_f, + xfs_lsn_t current_lsn) { struct xfs_da_blkinfo *info = bp->b_addr;__uint32_t magic32;
@@ -2569,7 +2570,8 @@ xlog_recover_do_reg_buffer(struct xfs_mount *mp, xlog_recover_item_t *item, struct xfs_buf *bp, - xfs_buf_log_format_t *buf_f) + xfs_buf_log_format_t *buf_f, + xfs_lsn_t current_lsn) { int i;int bit;
@@ -2642,7 +2644,7 @@ xlog_recover_do_reg_buffer(/* Shouldn't be any more regions */ ASSERT (i == item->ri_total); -xlog_recover_validate_buf_type (mp, bp, buf_f); +xlog_recover_validate_buf_type (mp, bp, buf_f, current_lsn); }/*
@@ -2685,7 +2687,7 @@ xlog_recover_do_dquot_buffer(if (log->l_quotaoffs_flag & type)return false; -xlog_recover_do_reg_buffer (mp, item, bp, buf_f); +xlog_recover_do_reg_buffer (mp, item, bp, buf_f,NULLCOMMITLSN );return true; }
@@ -2773,7 +2775,7 @@ xlog_recover_buffer_pass2(*/ lsn =xlog_recover_get_buf_lsn (mp, bp);if (lsn && lsn !=-1 &&XFS_LSN_CMP (lsn, current_lsn) >=0 ) { -xlog_recover_validate_buf_type (mp, bp, buf_f); +xlog_recover_validate_buf_type (mp, bp, buf_f,NULLCOMMITLSN ); goto out_release; }
@@ -2789,7 +2791,7 @@ xlog_recover_buffer_pass2(if (!dirty) goto out_release; }else { -xlog_recover_do_reg_buffer (mp, item, bp, buf_f); +xlog_recover_do_reg_buffer (mp, item, bp, buf_f, current_lsn); }/*
Signature · 1 line
2.5.5
PATCH v2 3/5 xfs: don't warn on buffers not being recovered due to LSN
In reply to Brian FosterThe log recovery buffer validation function is invoked in cases where a buffer update may be skipped due to LSN ordering. If the validation function happens to come across directory conversion situations (e.g., a dir3 block to data conversion), it may warn about seeing a buffer log format of one type and a buffer with a magic number of another.
This warning is not valid as the buffer update is ultimately skipped. This is indicated by a current_lsn of NULLCOMMITLSN provided by the caller. As such, update xlog_recover_validate_buf_type() to only warn in such cases when a buffer update is expected.
--- fs/xfs/xfs_log_recover.c | 58 ++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 32 deletions(-)
fs/xfs/xfs_log_recover.c +26 -32
@@ -2367,6 +2367,7 @@ xlog_recover_validate_buf_type(__uint32_t magic32;__uint16_t magic16;__uint16_t magicda; +char *warnmsg =NULL ;/* * We can only do post recovery validation on items on CRC enabled
@@ -2405,31 +2406,27 @@ xlog_recover_validate_buf_type( bp->b_ops = &xfs_rmapbt_buf_ops;break ;default : -xfs_warn (mp,"Bad btree block magic!" ); -ASSERT (0 ); + warnmsg ="Bad btree block magic!" ;break ; }break ;case XFS_BLFT_AGF_BUF :if (magic32 !=XFS_AGF_MAGIC ) { -xfs_warn (mp,"Bad AGF block magic!" ); -ASSERT (0 ); + warnmsg ="Bad AGF block magic!" ;break ; } bp->b_ops = &xfs_agf_buf_ops;break ;case XFS_BLFT_AGFL_BUF :if (magic32 !=XFS_AGFL_MAGIC ) { -xfs_warn (mp,"Bad AGFL block magic!" ); -ASSERT (0 ); + warnmsg ="Bad AGFL block magic!" ;break ; } bp->b_ops = &xfs_agfl_buf_ops;break ;case XFS_BLFT_AGI_BUF :if (magic32 !=XFS_AGI_MAGIC ) { -xfs_warn (mp,"Bad AGI block magic!" ); -ASSERT (0 ); + warnmsg ="Bad AGI block magic!" ;break ; } bp->b_ops = &xfs_agi_buf_ops;
@@ -2439,8 +2436,7 @@ xlog_recover_validate_buf_type(case XFS_BLFT_GDQUOT_BUF :#ifdef CONFIG_XFS_QUOTA if (magic16 !=XFS_DQUOT_MAGIC ) { -xfs_warn (mp,"Bad DQUOT block magic!" ); -ASSERT (0 ); + warnmsg ="Bad DQUOT block magic!" ;break ; } bp->b_ops = &xfs_dquot_buf_ops;
@@ -2452,16 +2448,14 @@ xlog_recover_validate_buf_type(break ;case XFS_BLFT_DINO_BUF :if (magic16 !=XFS_DINODE_MAGIC ) { -xfs_warn (mp,"Bad INODE block magic!" ); -ASSERT (0 ); + warnmsg ="Bad INODE block magic!" ;break ; } bp->b_ops = &xfs_inode_buf_ops;break ;case XFS_BLFT_SYMLINK_BUF :if (magic32 !=XFS_SYMLINK_MAGIC ) { -xfs_warn (mp,"Bad symlink block magic!" ); -ASSERT (0 ); + warnmsg ="Bad symlink block magic!" ;break ; } bp->b_ops = &xfs_symlink_buf_ops;
@@ -2469,8 +2463,7 @@ xlog_recover_validate_buf_type(case XFS_BLFT_DIR_BLOCK_BUF :if (magic32 !=XFS_DIR2_BLOCK_MAGIC && magic32 !=XFS_DIR3_BLOCK_MAGIC ) { -xfs_warn (mp,"Bad dir block magic!" ); -ASSERT (0 ); + warnmsg ="Bad dir block magic!" ;break ; } bp->b_ops = &xfs_dir3_block_buf_ops;
@@ -2478,8 +2471,7 @@ xlog_recover_validate_buf_type(case XFS_BLFT_DIR_DATA_BUF :if (magic32 !=XFS_DIR2_DATA_MAGIC && magic32 !=XFS_DIR3_DATA_MAGIC ) { -xfs_warn (mp,"Bad dir data magic!" ); -ASSERT (0 ); + warnmsg ="Bad dir data magic!" ;break ; } bp->b_ops = &xfs_dir3_data_buf_ops;
@@ -2487,8 +2479,7 @@ xlog_recover_validate_buf_type(case XFS_BLFT_DIR_FREE_BUF :if (magic32 !=XFS_DIR2_FREE_MAGIC && magic32 !=XFS_DIR3_FREE_MAGIC ) { -xfs_warn (mp,"Bad dir3 free magic!" ); -ASSERT (0 ); + warnmsg ="Bad dir3 free magic!" ;break ; } bp->b_ops = &xfs_dir3_free_buf_ops;
@@ -2496,8 +2487,7 @@ xlog_recover_validate_buf_type(case XFS_BLFT_DIR_LEAF1_BUF :if (magicda !=XFS_DIR2_LEAF1_MAGIC && magicda !=XFS_DIR3_LEAF1_MAGIC ) { -xfs_warn (mp,"Bad dir leaf1 magic!" ); -ASSERT (0 ); + warnmsg ="Bad dir leaf1 magic!" ;break ; } bp->b_ops = &xfs_dir3_leaf1_buf_ops;
@@ -2505,8 +2495,7 @@ xlog_recover_validate_buf_type(case XFS_BLFT_DIR_LEAFN_BUF :if (magicda !=XFS_DIR2_LEAFN_MAGIC && magicda !=XFS_DIR3_LEAFN_MAGIC ) { -xfs_warn (mp,"Bad dir leafn magic!" ); -ASSERT (0 ); + warnmsg ="Bad dir leafn magic!" ;break ; } bp->b_ops = &xfs_dir3_leafn_buf_ops;
@@ -2514,8 +2503,7 @@ xlog_recover_validate_buf_type(case XFS_BLFT_DA_NODE_BUF :if (magicda !=XFS_DA_NODE_MAGIC && magicda !=XFS_DA3_NODE_MAGIC ) { -xfs_warn (mp,"Bad da node magic!" ); -ASSERT (0 ); + warnmsg ="Bad da node magic!" ;break ; } bp->b_ops = &xfs_da3_node_buf_ops;
@@ -2523,24 +2511,21 @@ xlog_recover_validate_buf_type(case XFS_BLFT_ATTR_LEAF_BUF :if (magicda !=XFS_ATTR_LEAF_MAGIC && magicda !=XFS_ATTR3_LEAF_MAGIC ) { -xfs_warn (mp,"Bad attr leaf magic!" ); -ASSERT (0 ); + warnmsg ="Bad attr leaf magic!" ;break ; } bp->b_ops = &xfs_attr3_leaf_buf_ops;break ;case XFS_BLFT_ATTR_RMT_BUF :if (magic32 !=XFS_ATTR3_RMT_MAGIC ) { -xfs_warn (mp,"Bad attr remote magic!" ); -ASSERT (0 ); + warnmsg ="Bad attr remote magic!" ;break ; } bp->b_ops = &xfs_attr3_rmt_buf_ops;break ;case XFS_BLFT_SB_BUF :if (magic32 !=XFS_SB_MAGIC ) { -xfs_warn (mp,"Bad SB block magic!" ); -ASSERT (0 ); + warnmsg ="Bad SB block magic!" ;break ; } bp->b_ops = &xfs_sb_buf_ops;
@@ -2557,6 +2542,15 @@ xlog_recover_validate_buf_type( xfs_blft_from_flags(buf_f));break ; } + +/* +* Don't warn in the case of a NULL current LSN as this means the buffer +* is more recent than the change in the log and will be skipped. +*/ +if (warnmsg && current_lsn !=NULLCOMMITLSN ) { +xfs_warn (mp, warnmsg); +ASSERT (0 ); + } }/*
Signature · 1 line
2.5.5
PATCH v2 4/5 xfs: update metadata LSN in buffers during log recovery
In reply to Brian FosterLog recovery is currently broken for v5 superblocks in that it never updates the metadata LSN of buffers written out during recovery. The metadata LSN is recorded in various bits of metadata to provide recovery ordering criteria that prevents transient corruption states reported by buffer write verifiers. Without such ordering logic, buffer updates can be replayed out of order and lead to false positive transient corruption states. This is generally not a corruption vector on its own, but corruption detection shuts down the filesystem and ultimately prevents a mount if it occurs during log recovery. This requires an xfs_repair run that clears the log and potentially loses filesystem updates.
This problem is avoided in most cases as metadata writes during normal filesystem operation update the metadata LSN appropriately. The problem with log recovery not updating metadata LSNs manifests if the system happens to crash shortly after log recovery itself. In this scenario, it is possible for log recovery to complete all metadata I/O such that the filesystem is consistent. If a crash occurs after that point but before the log tail is pushed forward by subsequent operations, however, the next mount performs the same log recovery over again. If a buffer is updated multiple times in the dirty range of the log, an earlier update in the log might not be valid based on the current state of the associated buffer after all of the updates in the log had been replayed (before the previous crash). If a verifier happens to detect such a problem, the filesystem claims corruption and immediately shuts down.
This commonly manifests in practice as directory block verifier failures such as the following, likely due to directory verifiers being particularly detailed in their checks as compared to most others:
...
Mounting V5 Filesystem
XFS (dm-0): Starting recovery (logdev: internal)
XFS (dm-0): Internal error XFS_WANT_CORRUPTED_RETURN at line ... of \
file fs/xfs/libxfs/xfs_dir2_data.c. Caller xfs_dir3_data_verify ...
...
Update log recovery to update the metadata LSN of recovered buffers. Since metadata LSNs are already updated by write verifer functions via attached log items, attach a dummy log item to the buffer during validation and explicitly set the LSN of the current transaction. This ensures that the metadata LSN of a buffer is updated based on whether the recovery I/O actually completes, and if so, that subsequent recovery attempts identify that the buffer is already up to date with respect to the current transaction.
--- fs/xfs/xfs_log_recover.c | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-)
fs/xfs/xfs_log_recover.c +38 -3
@@ -44,6 +44,7 @@#include "xfs_error.h" #include "xfs_dir2.h" #include "xfs_rmap_item.h" +#include "xfs_buf_item.h" #define BLK_AVG (blk1, blk2) ((blk1+blk2) >> 1)
@@ -381,6 +382,15 @@ xlog_recover_iodone( SHUTDOWN_META_IO_ERROR); } } + +/* +* On v5 supers, a bli could be attached to update the metadata LSN. +* Clean it up. +*/ +if (bp->b_fspriv) +xfs_buf_item_relse (bp); +ASSERT (bp->b_fspriv ==NULL ); + bp->b_iodone =NULL ;xfs_buf_ioend (bp); }
@@ -2544,13 +2554,38 @@ xlog_recover_validate_buf_type( }/* -* Don't warn in the case of a NULL current LSN as this means the buffer -* is more recent than the change in the log and will be skipped. +* Nothing else to do in the case of a NULL current LSN as this means +* the buffer is more recent than the change in the log and will be +* skipped. */ -if (warnmsg && current_lsn !=NULLCOMMITLSN ) { +if (current_lsn ==NULLCOMMITLSN ) +return ; + +if (warnmsg) {xfs_warn (mp, warnmsg);ASSERT (0 ); } + +/* +* We must update the metadata LSN of the buffer as it is written out to +* ensure that older transactions never replay over this one and corrupt +* the buffer. This can occur if log recovery is interrupted at some +* point after the current transaction completes, at which point a +* subsequent mount starts recovery from the beginning. +* +* Write verifiers update the metadata LSN from log items attached to +* the buffer. Therefore, initialize a bli purely to carry the LSN to +* the verifier. We'll clean it up in our ->iodone() callback. +*/ +if (bp->b_ops) { +struct xfs_buf_log_item *bip; + +ASSERT (!bp->b_iodone || bp->b_iodone == xlog_recover_iodone); + bp->b_iodone = xlog_recover_iodone; +xfs_buf_item_init (bp, mp); + bip = bp->b_fspriv; + bip->bli_item.li_lsn = current_lsn; + } }/*
Signature · 1 line
2.5.5
PATCH v2 5/5 xfs: log recovery tracepoints to track current lsn and buffer submission
In reply to Brian FosterLog recovery has particular rules around buffer submission along with tricky corner cases where independent transactions can share an LSN. As such, it can be difficult to follow when/why buffers are submitted during recovery.
Add a couple tracepoints to post the current LSN of a record when a new record is being processed and when a buffer is being skipped due to LSN ordering. Also, update the recover item class to include the LSN of the current transaction for the item being processed.
--- fs/xfs/xfs_log_recover.c | 2 ++ fs/xfs/xfs_trace.h | 31 +++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-)
fs/xfs/xfs_log_recover.c +2 -0
@@ -2804,6 +2804,7 @@ xlog_recover_buffer_pass2(*/ lsn =xlog_recover_get_buf_lsn (mp, bp);if (lsn && lsn !=-1 &&XFS_LSN_CMP (lsn, current_lsn) >=0 ) { +trace_xfs_log_recover_buf_skip (log, buf_f);xlog_recover_validate_buf_type (mp, bp, buf_f,NULLCOMMITLSN ); goto out_release; }
@@ -4319,6 +4320,7 @@ xlog_recover_process_data(if (xlog_header_check_recover (log->l_mp, rhead))return -EIO ; +trace_xfs_log_recover_record (log, rhead, pass);while ((dp < end) && num_logops) { ohead = (struct xlog_op_header *)dp;
fs/xfs/xfs_trace.h +29 -2
@@ -1985,6 +1985,29 @@ DEFINE_EVENT(xfs_swap_extent_class, name, \DEFINE_SWAPEXT_EVENT (xfs_swap_extent_before);DEFINE_SWAPEXT_EVENT (xfs_swap_extent_after); +TRACE_EVENT (xfs_log_recover_record, + TP_PROTO(struct xlog *log, struct xlog_rec_header *rhead, int pass), +TP_ARGS (log, rhead, pass), +TP_STRUCT__entry ( + __field(dev_t, dev) + __field(xfs_lsn_t, lsn) + __field(int, len) + __field(int, num_logops) +__field (int, pass) + ), +TP_fast_assign ( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->lsn = be64_to_cpu(rhead->h_lsn); + __entry->len = be32_to_cpu(rhead->h_len); + __entry->num_logops = be32_to_cpu(rhead->h_num_logops); + __entry->pass = pass; + ), +TP_printk ("dev %d:%d lsn 0x%llx len 0x%x num_logops 0x%x pass %d" , +MAJOR (__entry->dev),MINOR (__entry->dev), + __entry->lsn, __entry->len, __entry->num_logops, + __entry->pass) +) + DECLARE_EVENT_CLASS(xfs_log_recover_item_class, TP_PROTO(struct xlog *log, struct xlog_recover *trans, struct xlog_recover_item *item, int pass),
@@ -1993,6 +2016,7 @@ DECLARE_EVENT_CLASS(xfs_log_recover_item_class, __field(dev_t, dev) __field(unsigned long, item) __field(xlog_tid_t, tid) + __field(xfs_lsn_t, lsn) __field(int, type) __field(int, pass) __field(int, count)
@@ -2002,15 +2026,17 @@ DECLARE_EVENT_CLASS(xfs_log_recover_item_class, __entry->dev = log->l_mp->m_super->s_dev; __entry->item = (unsigned long )item; __entry->tid = trans->r_log_tid; + __entry->lsn = trans->r_lsn; __entry->type =ITEM_TYPE (item); __entry->pass = pass; __entry->count = item->ri_cnt; __entry->total = item->ri_total; ), - TP_printk("dev %d:%d trans 0x%x, pass %d, item 0x%p, item type %s " - "item region count/total %d/%d", + TP_printk("dev %d:%d tid 0x%x lsn 0x%llx, pass %d, item 0x%p, " + "item type %s item region count/total %d/%d", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->tid, + __entry->lsn, __entry->pass, (void *)__entry->item, __print_symbolic(__entry->type, XFS_LI_TYPE_DESC),
@@ -2069,6 +2095,7 @@ DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_cancel);DEFINE_LOG_RECOVER_BUF_ITEM (xfs_log_recover_buf_cancel_add);DEFINE_LOG_RECOVER_BUF_ITEM (xfs_log_recover_buf_cancel_ref_inc);DEFINE_LOG_RECOVER_BUF_ITEM (xfs_log_recover_buf_recover); +DEFINE_LOG_RECOVER_BUF_ITEM (xfs_log_recover_buf_skip);DEFINE_LOG_RECOVER_BUF_ITEM (xfs_log_recover_buf_inode_buf);DEFINE_LOG_RECOVER_BUF_ITEM (xfs_log_recover_buf_reg_buf);DEFINE_LOG_RECOVER_BUF_ITEM (xfs_log_recover_buf_dquot_buf);
Signature · 1 line
2.5.5