cleanup and bugfix for xfs tests related to btree format
v3 cover cleanup and bugfix for xfs tests related to btree format
The first patch add a helper in common/xfs to export the inode core size which is needed by some xfs test cases.
The second patch ensure that S_IFDIR.FMT_BTREE is in btree format while populating dir.
V3:
- refactor xfs tests cases using inode core size
V2:
- take Darrick's advice to cleanup code
Ziyang Zhang (2):
common/xfs: Add a helper to export inode core size common/populate: Ensure that S_IFDIR.FMT_BTREE is in btree format
common/populate | 28 +++++++++++++++++++++++++++- common/xfs | 16 ++++++++++++++++ tests/xfs/335 | 3 ++- tests/xfs/336 | 3 ++- tests/xfs/337 | 3 ++- tests/xfs/341 | 3 ++- tests/xfs/342 | 3 ++- 7 files changed, 53 insertions(+), 6 deletions(-)
Signature · 1 line
2.18.4
PATCH v3 1/2 common/xfs: Add a helper to export inode core size
Some xfs test cases need the number of bytes reserved for only the inode record, excluding the immediate fork areas. Now the value is hard-coded and it is not a good chioce. Add a helper in common/xfs to export the inode core size.
--- common/xfs | 7 +++++++ tests/xfs/335 | 3 ++- tests/xfs/336 | 3 ++- tests/xfs/337 | 3 ++- tests/xfs/341 | 3 ++- tests/xfs/342 | 3 ++- 6 files changed, 17 insertions(+), 5 deletions(-)
common/xfs +7 -0
@@ -1486,3 +1486,10 @@ _require_xfsrestore_xflag() $XFSRESTORE_PROG -h 2>&1 | grep -q -e '-x' || \ _notrun 'xfsrestore does not support -x flag.' } + +# Number of bytes reserved for only the inode record, excluding the +# immediate fork areas. +_xfs_inode_core_bytes() +{ + echo 176 +}
tests/xfs/335 +2 -1
@@ -31,7 +31,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)" echo "Create a three-level rtrmapbt" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_ptrs=$(( (blksz - 56) / 56 )) bt_recs=$(( (blksz - 56) / 32 ))
tests/xfs/336 +2 -1
@@ -42,7 +42,8 @@ rm -rf $metadump_file echo "Create a three-level rtrmapbt" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_ptrs=$(( (blksz - 56) / 56 )) bt_recs=$(( (blksz - 56) / 32 ))
tests/xfs/337 +2 -1
@@ -33,7 +33,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_ptrs=$(( (blksz - 56) / 56 )) bt_recs=$(( (blksz - 56) / 32 ))
tests/xfs/341 +2 -1
@@ -33,7 +33,8 @@ rtextsz_blks=$((rtextsz / blksz)) # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_recs=$(( (blksz - 56) / 32 )) blocks=$((i_ptrs * bt_recs + 1))
tests/xfs/342 +2 -1
@@ -30,7 +30,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_recs=$(( (blksz - 56) / 32 )) blocks=$((i_ptrs * bt_recs + 1))
Signature · 1 line
2.18.4
Re: [PATCH V3 1/2] common/xfs: Add a helper to export inode core size
Quoted 6 lines · Ziyang Zhang
Some xfs test cases need the number of bytes reserved for only the inode record, excluding the immediate fork areas. Now the value is hard-coded and it is not a good chioce. Add a helper in common/xfs to export the inode core size.
Signed-off-by Ziyang Zhang <[email protected]>
Looks good,
--D
Quoted 97 lines
--- common/xfs | 7 +++++++ tests/xfs/335 | 3 ++- tests/xfs/336 | 3 ++- tests/xfs/337 | 3 ++- tests/xfs/341 | 3 ++- tests/xfs/342 | 3 ++- 6 files changed, 17 insertions(+), 5 deletions(-)common/xfs +7
-0@@ -1486,3 +1486,10 @@ _require_xfsrestore_xflag() $XFSRESTORE_PROG -h 2>&1 | grep -q -e '-x' || \ _notrun 'xfsrestore does not support -x flag.' } + +# Number of bytes reserved for only the inode record, excluding the +# immediate fork areas. +_xfs_inode_core_bytes() +{ + echo 176 +}tests/xfs/335 +2
-1@@ -31,7 +31,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)" echo "Create a three-level rtrmapbt" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_ptrs=$(( (blksz - 56) / 56 )) bt_recs=$(( (blksz - 56) / 32 ))tests/xfs/336 +2
-1@@ -42,7 +42,8 @@ rm -rf $metadump_file echo "Create a three-level rtrmapbt" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_ptrs=$(( (blksz - 56) / 56 )) bt_recs=$(( (blksz - 56) / 32 ))tests/xfs/337 +2
-1@@ -33,7 +33,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_ptrs=$(( (blksz - 56) / 56 )) bt_recs=$(( (blksz - 56) / 32 ))tests/xfs/341 +2
-1@@ -33,7 +33,8 @@ rtextsz_blks=$((rtextsz / blksz)) # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_recs=$(( (blksz - 56) / 32 )) blocks=$((i_ptrs * bt_recs + 1))tests/xfs/342 +2
-1@@ -30,7 +30,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)" # inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes. -i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 )) bt_recs=$(( (blksz - 56) / 32 )) blocks=$((i_ptrs * bt_recs + 1))Signature · 1 line
2.18.4
Re: [PATCH V3 1/2] common/xfs: Add a helper to export inode core size
In reply to Ziyang ZhangQuoted 8 lines · Ziyang Zhang
Some xfs test cases need the number of bytes reserved for only the inode
record, excluding the immediate fork areas. Now the value is hard- coded
and it is not a good chioce. Add a helper in common/xfs to export the inode core size.Signed-off-by Ziyang Zhang <[email protected]>
Looks clean to me
Quoted 95 lines
---
common/xfs | 7 +++++++tests/xfs/335 | 3 ++- tests/xfs/336 | 3 ++- tests/xfs/337 | 3 ++- tests/xfs/341 | 3 ++- tests/xfs/342 | 3 ++- 6 files changed, 17 insertions(+), 5 deletions(-)common/xfs +0
-0@@ -1486,3 +1486,10 @@ _require_xfsrestore_xflag() $XFSRESTORE_PROG -h 2>&1 | grep -q -e '-x' || \_notrun 'xfsrestore does not support -x flag.'
}common/xfs +7
-0+ +# Number of bytes reserved for only the inode record, excluding the +# immediate fork areas. +_xfs_inode_core_bytes() +{ + echo 176 +}tests/xfs/335 +0
-0@@ -31,7 +31,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)"echo "Create a three-level rtrmapbt"
# inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes.tests/xfs/335 +2
-1-i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 ))bt_ptrs=$(( (blksz - 56) / 56 ))
bt_recs=$(( (blksz - 56) / 32 ))tests/xfs/336 +0
-0@@ -42,7 +42,8 @@ rm -rf $metadump_fileecho "Create a three-level rtrmapbt"
# inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes.tests/xfs/336 +2
-1-i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 ))bt_ptrs=$(( (blksz - 56) / 56 ))
bt_recs=$(( (blksz - 56) / 32 ))tests/xfs/337 +0
-0@@ -33,7 +33,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)"# inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes.tests/xfs/337 +2
-1-i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 ))bt_ptrs=$(( (blksz - 56) / 56 ))
bt_recs=$(( (blksz - 56) / 32 ))tests/xfs/341 +0
-0@@ -33,7 +33,8 @@ rtextsz_blks=$((rtextsz / blksz))# inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes.tests/xfs/341 +2
-1-i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 ))bt_recs=$(( (blksz - 56) / 32 ))
blocks=$((i_ptrs *bt_recs+ 1))tests/xfs/342 +0
-0@@ -30,7 +30,8 @@ blksz="$(_get_block_size $SCRATCH_MNT)"# inode core size is at least 176 bytes; btree header is 56 bytes; # rtrmap record is 32 bytes; and rtrmap key/pointer are 56 bytes.tests/xfs/342 +2
-1-i_ptrs=$(( (isize - 176) / 56 )) +i_core_size="$(_xfs_inode_core_bytes)" +i_ptrs=$(( (isize - i_core_size) / 56 ))bt_recs=$(( (blksz - 56) / 32 ))
blocks=$((i_ptrs *bt_recs+ 1))
PATCH v3 2/2 common/populate: Ensure that S_IFDIR.FMT_BTREE is in btree format
In reply to Ziyang ZhangSometimes "$((128 * dblksz / 40))" dirents cannot make sure that S_IFDIR.FMT_BTREE could become btree format for its DATA fork.
Actually we just observed it can fail after apply our inode extent-to-btree workaround. The root cause is that the kernel may be too good at allocating consecutive blocks so that the data fork is still in extents format.
Therefore instead of using a fixed number, let's make sure the number of extents is large enough than (inode size - inode core size) / sizeof(xfs_bmbt_rec_t).
--- common/populate | 28 +++++++++++++++++++++++++++- common/xfs | 9 +++++++++ 2 files changed, 36 insertions(+), 1 deletion(-)
common/populate +27 -1
@@ -71,6 +71,31 @@ __populate_create_dir() { done } +# Create a large directory and ensure that it's a btree format +__populate_xfs_create_btree_dir() { + local name="$1" + local isize="$2" + local icore_size="$(_xfs_inode_core_bytes)" + # We need enough extents to guarantee that the data fork is in + # btree format. Cycling the mount to use xfs_db is too slow, so + # watch for when the extent count exceeds the space after the + # inode core. + local max_nextents="$(((isize - icore_size) / 16))" + + mkdir -p "${name}" + d=0 + while true; do + creat=mkdir + test "$((d % 20))" -eq 0 && creat=touch + $creat "${name}/$(printf "%.08d" "$d")" + if [ "$((d % 40))" -eq 0 ]; then + nextents="$(_xfs_get_fsxattr nextents $name)" + [ $nextents -gt $max_nextents ] && break + fi + d=$((d+1)) + done +} + # Add a bunch of attrs to a file __populate_create_attr() { name="$1"
@@ -176,6 +201,7 @@ _scratch_xfs_populate() { blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")" dblksz="$(_xfs_get_dir_blocksize "$SCRATCH_MNT")" + isize="$(_xfs_inode_size "$SCRATCH_MNT")" crc="$(_xfs_has_feature "$SCRATCH_MNT" crc -v)" if [ $crc -eq 1 ]; then leaf_hdr_size=64
@@ -226,7 +252,7 @@ _scratch_xfs_populate() { # - BTREE echo "+ btree dir" - __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true + __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize" # Symlinks # - FMT_LOCAL
common/xfs +9 -0
@@ -1487,6 +1487,15 @@ _require_xfsrestore_xflag() _notrun 'xfsrestore does not support -x flag.' } +# Number of bytes reserved for a full inode record, which includes the +# immediate fork areas. +_xfs_inode_size() +{ + local mntpoint="$1" + + $XFS_INFO_PROG "$mntpoint" | grep 'meta-data=.*isize' | sed -e 's/^.*isize=\([0-9]*\).*$/\1/g' +} + # Number of bytes reserved for only the inode record, excluding the # immediate fork areas. _xfs_inode_core_bytes()
Signature · 1 line
2.18.4
Re: [PATCH V3 2/2] common/populate: Ensure that S_IFDIR.FMT_BTREE is in btree format
Quoted 66 lines · Ziyang Zhang
Sometimes "$((128 * dblksz / 40))" dirents cannot make sure that S_IFDIR.FMT_BTREE could become btree format for its DATA fork.
Actually we just observed it can fail after apply our inode extent-to-btree workaround. The root cause is that the kernel may be too good at allocating consecutive blocks so that the data fork is still in extents format.
Therefore instead of using a fixed number, let's make sure the number of extents is large enough than (inode size - inode core size) /
sizeof(xfs_bmbt_rec_t).Reviewed-by Allison Henderson <[email protected]>Suggested-by "Darrick J. Wong" <[email protected]>Signed-off-by Gao Xiang <[email protected]>Signed-off-by Ziyang Zhang <[email protected]>--- common/populate | 28 +++++++++++++++++++++++++++- common/xfs | 9 +++++++++ 2 files changed, 36 insertions(+), 1 deletion(-)common/populate +26
-0@@ -71,6 +71,31 @@ __populate_create_dir() { done } +# Create a large directory and ensure that it's a btree format +__populate_xfs_create_btree_dir() { + local name="$1" + local isize="$2" + local icore_size="$(_xfs_inode_core_bytes)" + # We need enough extents to guarantee that the data fork is in + # btree format. Cycling the mount to use xfs_db is too slow, so + # watch for when the extent count exceeds the space after the + # inode core. + local max_nextents="$(((isize - icore_size) / 16))" + + mkdir -p "${name}" + d=0 + while true; do + creat=mkdir + test "$((d % 20))" -eq 0 && creat=touch + $creat "${name}/$(printf "%.08d" "$d")" + if [ "$((d % 40))" -eq 0 ]; then + nextents="$(_xfs_get_fsxattr nextents $name)" + [ $nextents -gt $max_nextents ] && break + fi + d=$((d+1)) + done +} + # Add a bunch of attrs to a file __populate_create_attr() { name="$1"@@ -176,6 +201,7 @@ _scratch_xfs_populate() { blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")" dblksz="$(_xfs_get_dir_blocksize "$SCRATCH_MNT")" + isize="$(_xfs_inode_size "$SCRATCH_MNT")" crc="$(_xfs_has_feature "$SCRATCH_MNT" crc -v)" if [ $crc -eq 1 ]; then leaf_hdr_size=64
common/populate _scratch_xfs_populate() { · line 255
Quoted 5 lines
common/populate +1
-1@@ -226,7 +252,7 @@ _scratch_xfs_populate() { # - BTREE echo "+ btree dir" - __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true + __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize"
The new helper function omits the "missing" parameter, which means that it no longer creates the directory entry blocks with a lot of free space in them, unlike current TOT.
--D
Quoted 26 lines
common/populate +0
-0# Symlinks # - FMT_LOCALcommon/xfs +9
-0@@ -1487,6 +1487,15 @@ _require_xfsrestore_xflag() _notrun 'xfsrestore does not support -x flag.' } +# Number of bytes reserved for a full inode record, which includes the +# immediate fork areas. +_xfs_inode_size() +{ + local mntpoint="$1" + + $XFS_INFO_PROG "$mntpoint" | grep 'meta-data=.*isize' | sed -e 's/^.*isize=\([0-9]*\).*$/\1/g' +} + # Number of bytes reserved for only the inode record, excluding the # immediate fork areas. _xfs_inode_core_bytes()Signature · 1 line
2.18.4
Re: [PATCH V3 2/2] common/populate: Ensure that S_IFDIR.FMT_BTREE is in btree format
Quoted 68 lines · Darrick J. Wong
Quoted 67 lines · Ziyang Zhang
Sometimes "$((128 * dblksz / 40))" dirents cannot make sure that S_IFDIR.FMT_BTREE could become btree format for its DATA fork.
Actually we just observed it can fail after apply our inode extent-to-btree workaround. The root cause is that the kernel may be too good at allocating consecutive blocks so that the data fork is still in extents format.
Therefore instead of using a fixed number, let's make sure the number of extents is large enough than (inode size - inode core size) /
sizeof(xfs_bmbt_rec_t).Reviewed-by Allison Henderson <[email protected]>Suggested-by "Darrick J. Wong" <[email protected]>Signed-off-by Gao Xiang <[email protected]>Signed-off-by Ziyang Zhang <[email protected]>--- common/populate | 28 +++++++++++++++++++++++++++- common/xfs | 9 +++++++++ 2 files changed, 36 insertions(+), 1 deletion(-)common/populate +26
-0@@ -71,6 +71,31 @@ __populate_create_dir() { done } +# Create a large directory and ensure that it's a btree format +__populate_xfs_create_btree_dir() { + local name="$1" + local isize="$2" + local icore_size="$(_xfs_inode_core_bytes)" + # We need enough extents to guarantee that the data fork is in + # btree format. Cycling the mount to use xfs_db is too slow, so + # watch for when the extent count exceeds the space after the + # inode core. + local max_nextents="$(((isize - icore_size) / 16))" + + mkdir -p "${name}" + d=0 + while true; do + creat=mkdir + test "$((d % 20))" -eq 0 && creat=touch + $creat "${name}/$(printf "%.08d" "$d")" + if [ "$((d % 40))" -eq 0 ]; then + nextents="$(_xfs_get_fsxattr nextents $name)" + [ $nextents -gt $max_nextents ] && break + fi + d=$((d+1)) + done +} + # Add a bunch of attrs to a file __populate_create_attr() { name="$1"@@ -176,6 +201,7 @@ _scratch_xfs_populate() { blksz="$(stat -f -c '%s' "${SCRATCH_MNT}")" dblksz="$(_xfs_get_dir_blocksize "$SCRATCH_MNT")" + isize="$(_xfs_inode_size "$SCRATCH_MNT")" crc="$(_xfs_has_feature "$SCRATCH_MNT" crc -v)" if [ $crc -eq 1 ]; then leaf_hdr_size=64@@ -226,7 +252,7 @@ _scratch_xfs_populate() {
Quoted 8 lines
common/populate _scratch_xfs_populate() { · line 255
Quoted 4 lines
common/populate +1
-1# - BTREE echo "+ btree dir" - __populate_create_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$((128 * dblksz / 40))" true + __populate_xfs_create_btree_dir "${SCRATCH_MNT}/S_IFDIR.FMT_BTREE" "$isize"The new helper function omits the "missing" parameter, which means that it no longer creates the directory entry blocks with a lot of free space in them, unlike current TOT.
Hi Ziyang,
Is there any reason we must drop this "missing" step. If no special reason, I'd like to keep the original operation (the missing parameter is "true") to free some space sparsely. I think that only removes some entries, won't remove the dir-blocks (generally), so it won't affect the btree format you create.
Thanks,
Zorro
Quoted 30 lines
--D
Quoted 26 lines
common/populate +0
-0# Symlinks # - FMT_LOCALcommon/xfs +9
-0@@ -1487,6 +1487,15 @@ _require_xfsrestore_xflag() _notrun 'xfsrestore does not support -x flag.' } +# Number of bytes reserved for a full inode record, which includes the +# immediate fork areas. +_xfs_inode_size() +{ + local mntpoint="$1" + + $XFS_INFO_PROG "$mntpoint" | grep 'meta-data=.*isize' | sed -e 's/^.*isize=\([0-9]*\).*$/\1/g' +} + # Number of bytes reserved for only the inode record, excluding the # immediate fork areas. _xfs_inode_core_bytes()Signature · 1 line
2.18.4