libfrog: hoist bitmap out of scrub
PATCH 1/2 libfrog: hoist bitmap out of scrub
From: Darrick J. Wong <[email protected]>
Move the bitmap code to libfrog so that we can use bitmaps in xfs_repair.
--- include/bitmap.h | 24 +++ libfrog/Makefile | 1 libfrog/bitmap.c | 393 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ scrub/Makefile | 2 scrub/bitmap.c | 393 ------------------------------------------------------ scrub/bitmap.h | 24 --- 6 files changed, 418 insertions(+), 419 deletions(-) create mode 100644 include/bitmap.h create mode 100644 libfrog/bitmap.c delete mode 100644 scrub/bitmap.c delete mode 100644 scrub/bitmap.h
include/bitmap.h new file +24 -0
@@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* +* Copyright (C) 2018 Oracle. All Rights Reserved. +* Author: Darrick J. Wong <[email protected]> +*/ +#ifndef LIBFROG_BITMAP_H_ +#define LIBFROG_BITMAP_H_ + +struct bitmap { +pthread_mutex_t bt_lock; +struct avl64tree_desc *bt_tree; +}; + +bool bitmap_init (struct bitmap **bmap); +void bitmap_free (struct bitmap **bmap); +bool bitmap_set (struct bitmap *bmap,uint64_t start,uint64_t length); +bool bitmap_iterate (struct bitmap *bmap, +bool (*fn)(uint64_t ,uint64_t ,void *),void *arg); +bool bitmap_test (struct bitmap *bmap,uint64_t start, +uint64_t len); +bool bitmap_empty (struct bitmap *bmap); +void bitmap_dump (struct bitmap *bmap); + +#endif /* LIBFROG_BITMAP_H_ */
libfrog/Makefile +1 -0
@@ -12,6 +12,7 @@ LT_AGE = 0 CFILES = \ avl64.c \ +bitmap.c \ convert.c \ crc32.c \ fsgeom.c \
libfrog/bitmap.c new file +393 -0
393 lines not on this page · raw message
scrub/Makefile +0 -2
@@ -31,7 +31,6 @@ endif endif # scrub_prereqs HFILES = \ -bitmap.h \ common.h \ counter.h \ disk.h \
@@ -48,7 +47,6 @@ vfs.h \ xfs_scrub.h CFILES = \ -bitmap.c \ common.c \ counter.c \ disk.c \
scrub/bitmap.c deleted +0 -393
393 lines not on this page · raw message
scrub/bitmap.h deleted +0 -24
@@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* -* Copyright (C) 2018 Oracle. All Rights Reserved. -* Author: Darrick J. Wong <[email protected]> -*/ -#ifndef XFS_SCRUB_BITMAP_H_ -#define XFS_SCRUB_BITMAP_H_ - -struct bitmap { -pthread_mutex_t bt_lock; -struct avl64tree_desc *bt_tree; -}; - -bool bitmap_init (struct bitmap **bmap); -void bitmap_free (struct bitmap **bmap); -bool bitmap_set (struct bitmap *bmap,uint64_t start,uint64_t length); -bool bitmap_iterate (struct bitmap *bmap, -bool (*fn)(uint64_t ,uint64_t ,void *),void *arg); -bool bitmap_test (struct bitmap *bmap,uint64_t start, -uint64_t len); -bool bitmap_empty (struct bitmap *bmap); -void bitmap_dump (struct bitmap *bmap); - -#endif /* XFS_SCRUB_BITMAP_H_ */
PATCH 2/2 xfs_repair: correctly account for free space btree shrinks when fixing freelist
From: Darrick J. Wong <[email protected]>
When we fix the freelist at the end of build_agf_agfl in phase 5 of repair, we need to create incore rmap records for the blocks that get added to the AGFL. We can't let the regular freelist fixing code use the regular on-disk rmapbt update code because the rmapbt isn't fully set up yet.
Unfortunately, the original code fails to account for the fact that the free space btrees can shrink when we allocate blocks to fix the freelist; those blocks are also put on the freelist, but there are already incore rmaps for all the free space btree blocks. We must not create (redundant) incore rmaps for those blocks. If we do, repair fails with a complaint that rebuilding the rmapbt failed during phase 5. xfs/137 on a 1k block size occasionally triggers this bug.
To fix the problem, construct a bitmap of all OWN_AG blocks that we know about before traversing the AGFL, and only create new incore rmaps for those AGFL blocks that are not already tracked in the bitmap.
--- repair/rmap.c | 54 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-)
repair/rmap.c +44 -10
@@ -12,6 +12,7 @@#include "dinode.h" #include "slab.h" #include "rmap.h" +#include "bitmap.h" #undef RMAP_DEBUG
@@ -450,6 +451,8 @@ rmap_store_ag_btree_rec(struct xfs_buf *agflbp =NULL ;struct xfs_trans *tp;__be32 *agfl_bno, *b; +struct xfs_ag_rmap *ag_rmap = &ag_rmaps[agno]; +struct bitmap *own_ag_bitmap =NULL ;int error =0 ;struct xfs_owner_info oinfo;
@@ -457,9 +460,8 @@ rmap_store_ag_btree_rec(return 0 ;/* Release the ar_rmaps; they were put into the rmapbt during p5. */ -free_slab (&ag_rmaps[agno].ar_rmaps); - error =init_slab (&ag_rmaps[agno].ar_rmaps, -sizeof (struct xfs_rmap_irec )); +free_slab (&ag_rmap->ar_rmaps); + error =init_slab (&ag_rmap->ar_rmaps,sizeof (struct xfs_rmap_irec ));if (error) goto err;
@@ -479,19 +481,50 @@ rmap_store_ag_btree_rec(* rmap, we only need to add rmap records for AGFL blocks past * that point in the AGFL because those blocks are a result of a * no-rmap no-shrink freelist fixup that we did earlier. +* +* However, some blocks end up on the AGFL because the free space +* btrees shed blocks as a result of allocating space to fix the +* freelist. We already created in-core rmap records for the free +* space btree blocks, so we must be careful not to create those +* records again. Create a bitmap of already-recorded OWN_AG rmaps. */ + error =init_slab_cursor (ag_rmap->ar_raw_rmaps, rmap_compare, &rm_cur); +if (error) + goto err; +if (!bitmap_init (&own_ag_bitmap)) { + error = -ENOMEM ; + goto err; + } +while ((rm_rec =pop_slab_cursor (rm_cur)) !=NULL ) { +if (rm_rec->rm_owner !=XFS_RMAP_OWN_AG ) +continue ; +if (!bitmap_set (own_ag_bitmap, rm_rec->rm_startblock, + rm_rec->rm_blockcount)) { + error =EFSCORRUPTED ; + goto err; + } + } +free_slab_cursor (&rm_cur); + +/* Create rmaps for any AGFL blocks that aren't already rmapped. */ agfl_bno =XFS_BUF_TO_AGFL_BNO (mp, agflbp); - b = agfl_bno + ag_rmaps[agno].ar_flcount; + b = agfl_bno + ag_rmap->ar_flcount;while (*b !=cpu_to_be32 (NULLAGBLOCK ) && b - agfl_bno <libxfs_agfl_size (mp)) { - error =rmap_add_ag_rec (mp, agno,be32_to_cpu (*b),1 , -XFS_RMAP_OWN_AG ); -if (error) - goto err; +xfs_agblock_t agbno; + + agbno =be32_to_cpu (*b); +if (!bitmap_test (own_ag_bitmap, agbno,1 )) { + error =rmap_add_ag_rec (mp, agno, agbno,1 , +XFS_RMAP_OWN_AG ); +if (error) + goto err; + } b++; }libxfs_putbuf (agflbp); agflbp =NULL ; +bitmap_free (&own_ag_bitmap);/* Merge all the raw rmaps into the main list */ error =rmap_fold_raw_recs (mp, agno);
@@ -499,8 +532,7 @@ rmap_store_ag_btree_rec( goto err;/* Create cursors to refcount structures */ - error =init_slab_cursor (ag_rmaps[agno].ar_rmaps, rmap_compare, - &rm_cur); + error =init_slab_cursor (ag_rmap->ar_rmaps, rmap_compare, &rm_cur);if (error) goto err;
@@ -541,6 +573,8 @@ rmap_store_ag_btree_rec( err:if (agflbp)libxfs_putbuf (agflbp); +if (own_ag_bitmap) +bitmap_free (&own_ag_bitmap);return error; }
Re: [PATCH 2/2] xfs_repair: correctly account for free space btree shrinks when fixing freelist
Quoted 66 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
When we fix the freelist at the end of
build_agf_agflin phase 5 of repair, we need to create incore rmap records for the blocks that get added to the AGFL. We can't let the regular freelist fixing code use the regular on-disk rmapbt update code because the rmapbt isn't fully set up yet.Unfortunately, the original code fails to account for the fact that the free space btrees can shrink when we allocate blocks to fix the freelist; those blocks are also put on the freelist, but there are already incore rmaps for all the free space btree blocks. We must not create (redundant) incore rmaps for those blocks. If we do, repair fails with a complaint that rebuilding the rmapbt failed during phase 5. xfs/137 on a 1k block size occasionally triggers this bug.
To fix the problem, construct a bitmap of all
OWN_AGblocks that we know about before traversing the AGFL, and only create new incore rmaps for those AGFL blocks that are not already tracked in the bitmap.Signed-off-by Darrick J. Wong <[email protected]>--- repair/rmap.c | 54 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-)repair/rmap.c +8
-3@@ -12,6 +12,7 @@#include "dinode.h" #include "slab.h" #include "rmap.h" +#include "bitmap.h" #undef RMAP_DEBUG@@ -450,6 +451,8 @@ rmap_store_ag_btree_rec(struct xfs_buf *agflbp =NULL ;struct xfs_trans *tp;__be32 *agfl_bno, *b; +struct xfs_ag_rmap *ag_rmap = &ag_rmaps[agno]; +struct bitmap *own_ag_bitmap =NULL ;int error =0 ;struct xfs_owner_info oinfo;@@ -457,9 +460,8 @@ rmap_store_ag_btree_rec(return 0 ;/* Release the ar_rmaps; they were put into the rmapbt during p5. */ -free_slab (&ag_rmaps[agno].ar_rmaps); - error =init_slab (&ag_rmaps[agno].ar_rmaps, -sizeof (struct xfs_rmap_irec )); +free_slab (&ag_rmap->ar_rmaps); + error =init_slab (&ag_rmap->ar_rmaps,sizeof (struct xfs_rmap_irec ));if (error) goto err;@@ -479,19 +481,50 @@ rmap_store_ag_btree_rec(* rmap, we only need to add rmap records for AGFL blocks past * that point in the AGFL because those blocks are a result of a * no-rmap no-shrink freelist fixup that we did earlier. +* +* However, some blocks end up on the AGFL because the free space +* btrees shed blocks as a result of allocating space to fix the
repair/rmap.c rmap_store_ag_btree_rec( · line 496
Quoted 10 lines
repair/rmap.c +9
-0+* freelist. We already created in-core rmap records for the free +* space btree blocks, so we must be careful not to create those +* records again. Create a bitmap of already-recorded OWN_AG rmaps. */ + error =init_slab_cursor (ag_rmap->ar_raw_rmaps, rmap_compare, &rm_cur); +if (error) + goto err; +if (!bitmap_init (&own_ag_bitmap)) { + error = -ENOMEM ; + goto err;
I think that this ^^
repair/rmap.c rmap_store_ag_btree_rec( · line 504
Quoted 8 lines
repair/rmap.c +8
-0+ } +while ((rm_rec =pop_slab_cursor (rm_cur)) !=NULL ) { +if (rm_rec->rm_owner !=XFS_RMAP_OWN_AG ) +continue ; +if (!bitmap_set (own_ag_bitmap, rm_rec->rm_startblock, + rm_rec->rm_blockcount)) { + error =EFSCORRUPTED ; + goto err;
and this need to free up rm_cur to be tidy, right?
-Eric
Re: [PATCH 2/2] xfs_repair: correctly account for free space btree shrinks when fixing freelist
Quoted 80 lines · Eric Sandeen
Quoted 66 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
When we fix the freelist at the end of
build_agf_agflin phase 5 of repair, we need to create incore rmap records for the blocks that get added to the AGFL. We can't let the regular freelist fixing code use the regular on-disk rmapbt update code because the rmapbt isn't fully set up yet.Unfortunately, the original code fails to account for the fact that the free space btrees can shrink when we allocate blocks to fix the freelist; those blocks are also put on the freelist, but there are already incore rmaps for all the free space btree blocks. We must not create (redundant) incore rmaps for those blocks. If we do, repair fails with a complaint that rebuilding the rmapbt failed during phase 5. xfs/137 on a 1k block size occasionally triggers this bug.
To fix the problem, construct a bitmap of all
OWN_AGblocks that we know about before traversing the AGFL, and only create new incore rmaps for those AGFL blocks that are not already tracked in the bitmap.Signed-off-by Darrick J. Wong <[email protected]>--- repair/rmap.c | 54 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-)repair/rmap.c +8
-3@@ -12,6 +12,7 @@#include "dinode.h" #include "slab.h" #include "rmap.h" +#include "bitmap.h" #undef RMAP_DEBUG@@ -450,6 +451,8 @@ rmap_store_ag_btree_rec(struct xfs_buf *agflbp =NULL ;struct xfs_trans *tp;__be32 *agfl_bno, *b; +struct xfs_ag_rmap *ag_rmap = &ag_rmaps[agno]; +struct bitmap *own_ag_bitmap =NULL ;int error =0 ;struct xfs_owner_info oinfo;@@ -457,9 +460,8 @@ rmap_store_ag_btree_rec(return 0 ;/* Release the ar_rmaps; they were put into the rmapbt during p5. */ -free_slab (&ag_rmaps[agno].ar_rmaps); - error =init_slab (&ag_rmaps[agno].ar_rmaps, -sizeof (struct xfs_rmap_irec )); +free_slab (&ag_rmap->ar_rmaps); + error =init_slab (&ag_rmap->ar_rmaps,sizeof (struct xfs_rmap_irec ));if (error) goto err;@@ -479,19 +481,50 @@ rmap_store_ag_btree_rec(* rmap, we only need to add rmap records for AGFL blocks past * that point in the AGFL because those blocks are a result of a * no-rmap no-shrink freelist fixup that we did earlier. +* +* However, some blocks end up on the AGFL because the free space +* btrees shed blocks as a result of allocating space to fix the repair/rmap.c rmap_store_ag_btree_rec( · line 496
Quoted 10 lines
repair/rmap.c +9
-0+* freelist. We already created in-core rmap records for the free +* space btree blocks, so we must be careful not to create those +* records again. Create a bitmap of already-recorded OWN_AG rmaps. */ + error =init_slab_cursor (ag_rmap->ar_raw_rmaps, rmap_compare, &rm_cur); +if (error) + goto err; +if (!bitmap_init (&own_ag_bitmap)) { + error = -ENOMEM ; + goto err;I think that this ^^
Quoted 10 lines
repair/rmap.c rmap_store_ag_btree_rec( · line 504
Quoted 8 lines
repair/rmap.c +8
-0+ } +while ((rm_rec =pop_slab_cursor (rm_cur)) !=NULL ) { +if (rm_rec->rm_owner !=XFS_RMAP_OWN_AG ) +continue ; +if (!bitmap_set (own_ag_bitmap, rm_rec->rm_startblock, + rm_rec->rm_blockcount)) { + error =EFSCORRUPTED ; + goto err;and this need to free up
rm_curto be tidy, right?
Yep, they do need to be 'goto err_slab', thanks for catching that.
--D
Quoted 1 line
-Eric
PATCH v2 2/2 xfs_repair: correctly account for free space btree shrinks when fixing freelist
In reply to Darrick J. WongFrom: Darrick J. Wong <[email protected]>
When we fix the freelist at the end of build_agf_agfl in phase 5 of repair, we need to create incore rmap records for the blocks that get added to the AGFL. We can't let the regular freelist fixing code use the regular on-disk rmapbt update code because the rmapbt isn't fully set up yet.
Unfortunately, the original code fails to account for the fact that the free space btrees can shrink when we allocate blocks to fix the freelist; those blocks are also put on the freelist, but there are already incore rmaps for all the free space btree blocks. We must not create (redundant) incore rmaps for those blocks. If we do, repair fails with a complaint that rebuilding the rmapbt failed during phase 5. xfs/137 on a 1k block size occasionally triggers this bug.
To fix the problem, construct a bitmap of all OWN_AG blocks that we know about before traversing the AGFL, and only create new incore rmaps for those AGFL blocks that are not already tracked in the bitmap.
---
v2: fix error jumps
--- repair/rmap.c | 54 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-)
repair/rmap.c +44 -10
@@ -12,6 +12,7 @@#include "dinode.h" #include "slab.h" #include "rmap.h" +#include "bitmap.h" #undef RMAP_DEBUG
@@ -450,6 +451,8 @@ rmap_store_ag_btree_rec(struct xfs_buf *agflbp =NULL ;struct xfs_trans *tp;__be32 *agfl_bno, *b; +struct xfs_ag_rmap *ag_rmap = &ag_rmaps[agno]; +struct bitmap *own_ag_bitmap =NULL ;int error =0 ;struct xfs_owner_info oinfo;
@@ -457,9 +460,8 @@ rmap_store_ag_btree_rec(return 0 ;/* Release the ar_rmaps; they were put into the rmapbt during p5. */ -free_slab (&ag_rmaps[agno].ar_rmaps); - error =init_slab (&ag_rmaps[agno].ar_rmaps, -sizeof (struct xfs_rmap_irec )); +free_slab (&ag_rmap->ar_rmaps); + error =init_slab (&ag_rmap->ar_rmaps,sizeof (struct xfs_rmap_irec ));if (error) goto err;
@@ -479,19 +481,50 @@ rmap_store_ag_btree_rec(* rmap, we only need to add rmap records for AGFL blocks past * that point in the AGFL because those blocks are a result of a * no-rmap no-shrink freelist fixup that we did earlier. +* +* However, some blocks end up on the AGFL because the free space +* btrees shed blocks as a result of allocating space to fix the +* freelist. We already created in-core rmap records for the free +* space btree blocks, so we must be careful not to create those +* records again. Create a bitmap of already-recorded OWN_AG rmaps. */ + error =init_slab_cursor (ag_rmap->ar_raw_rmaps, rmap_compare, &rm_cur); +if (error) + goto err; +if (!bitmap_init (&own_ag_bitmap)) { + error = -ENOMEM ; + goto err_slab; + } +while ((rm_rec =pop_slab_cursor (rm_cur)) !=NULL ) { +if (rm_rec->rm_owner !=XFS_RMAP_OWN_AG ) +continue ; +if (!bitmap_set (own_ag_bitmap, rm_rec->rm_startblock, + rm_rec->rm_blockcount)) { + error =EFSCORRUPTED ; + goto err_slab; + } + } +free_slab_cursor (&rm_cur); + +/* Create rmaps for any AGFL blocks that aren't already rmapped. */ agfl_bno =XFS_BUF_TO_AGFL_BNO (mp, agflbp); - b = agfl_bno + ag_rmaps[agno].ar_flcount; + b = agfl_bno + ag_rmap->ar_flcount;while (*b !=cpu_to_be32 (NULLAGBLOCK ) && b - agfl_bno <libxfs_agfl_size (mp)) { - error =rmap_add_ag_rec (mp, agno,be32_to_cpu (*b),1 , -XFS_RMAP_OWN_AG ); -if (error) - goto err; +xfs_agblock_t agbno; + + agbno =be32_to_cpu (*b); +if (!bitmap_test (own_ag_bitmap, agbno,1 )) { + error =rmap_add_ag_rec (mp, agno, agbno,1 , +XFS_RMAP_OWN_AG ); +if (error) + goto err; + } b++; }libxfs_putbuf (agflbp); agflbp =NULL ; +bitmap_free (&own_ag_bitmap);/* Merge all the raw rmaps into the main list */ error =rmap_fold_raw_recs (mp, agno);
@@ -499,8 +532,7 @@ rmap_store_ag_btree_rec( goto err;/* Create cursors to refcount structures */ - error =init_slab_cursor (ag_rmaps[agno].ar_rmaps, rmap_compare, - &rm_cur); + error =init_slab_cursor (ag_rmap->ar_rmaps, rmap_compare, &rm_cur);if (error) goto err;
@@ -541,6 +573,8 @@ rmap_store_ag_btree_rec( err:if (agflbp)libxfs_putbuf (agflbp); +if (own_ag_bitmap) +bitmap_free (&own_ag_bitmap);return error; }