xfs: fix log reservation overflows when allocating large rt extents
PATCH 1/1 xfs: fix log reservation overflows when allocating large rt extents
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the tr_write and tr_itruncate log reservation calculation.
"The following reproducer results in a transaction log overrun warning for me:
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0 /dev/vdb mount -o rtdev=/dev/vdc /dev/vdb /mnt
fallocate -l 4G /mnt/foo
--- fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)
fs/xfs/libxfs/xfs_trans_resv.c +77 -19
@@ -196,6 +196,24 @@ xfs_calc_inode_chunk_res(return res; } +/* +* Per-extent log reservation for the btree changes involved in freeing or +* allocating a realtime extent. We have to be able to log as many rtbitmap +* blocks as needed to mark inuse MAXEXTLEN blocks' worth of realtime extents, +* as well as the realtime summary block. +*/ +unsigned int +xfs_rtalloc_log_count ( +struct xfs_mount *mp, +unsigned int num_ops) +{ +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); +unsigned int rtbmp_bytes; + + rtbmp_bytes = (MAXEXTLEN / mp->m_sb.sb_rextsize) /NBBY ; +return (howmany (rtbmp_bytes, blksz) +1 ) * num_ops; +} +/* * Various log reservation values. *
@@ -218,13 +236,21 @@ xfs_calc_inode_chunk_res(/* * In a write transaction we can allocate a maximum of 2 -* extents. This gives: +* extents. This gives (t1): * the inode getting the new extents: inode size * the inode's bmap btree: max depth * block size * the agfs of the ags from which the extents are allocated: 2 * sector * the superblock free block counter: sector size * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size -* And the bmap_finish transaction can free bmap blocks in a join: +* Or, if we're writing to a realtime file (t2): +* the inode getting the new extents: inode size +* the inode's bmap btree: max depth * block size +* the agfs of the ags from which the extents are allocated: 2 * sector +* the superblock free block counter: sector size +* the realtime bitmap: ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 1 block +* the allocation btrees: 2 trees * (2 * max depth - 1) * block size +* And the bmap_finish transaction can free bmap blocks in a join (t3): * the agfs of the ags containing the blocks: 2 * sector size * the agfls of the ags containing the blocks: 2 * sector size * the super block free block counter: sector size
@@ -234,40 +260,72 @@ STATIC uintxfs_calc_write_reservation ( struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), blksz) + +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t2 =xfs_calc_inode_res (mp,1 ) +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), -XFS_FSB_TO_B (mp,1 )) + + blksz) +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 )))); +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,1 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,1 ), blksz); + }else { + t2 =0 ; + } + + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/* -* In truncating a file we free up to two extents at once. We can modify: +* In truncating a file we free up to two extents at once. We can modify (t1): * the inode being truncated: inode size * the inode's bmap btree: (max depth + 1) * block size -* And the bmap_finish transaction can free the blocks and bmap blocks: +* And the bmap_finish transaction can free the blocks and bmap blocks (t2): * the agf for each of the ags: 4 * sector size * the agfl for each of the ags: 4 * sector size * the super block to reflect the freed blocks: sector size * worst case split in allocation btrees per extent assuming 4 extents: * 4 exts * 2 trees * (2 * max depth - 1) * block size +* Or, if it's a realtime file (t3): +* the agf for each of the ags: 2 * sector size +* the agfl for each of the ags: 2 * sector size +* the super block to reflect the freed blocks: sector size +* the realtime bitmap: 2 exts * ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 2 exts * 1 block +* worst case split in allocation btrees per extent assuming 2 extents: +* 2 exts * 2 trees * (2 * max depth - 1) * block size */ STATIC uintxfs_calc_itruncate_reservation (struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + -xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), -XFS_FSB_TO_B (mp,1 )))); +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , blksz); + + t2 =xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,2 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + }else { + t3 =0 ; + } + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/*
Quoted 7 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the
tr_writeandtr_itruncatelog reservation calculation.
Quoted 8 lines
"The following reproducer results in a transaction log overrun warning for me:
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0
/dev/vdbmount -o rtdev=/dev/vdc/dev/vdb/mnt
fallocate -l 4G/mnt/fooReported-by Omar Sandoval <[email protected]>
This one works, as well. Thanks!
Quoted 4 lines
Signed-off-by Darrick J. Wong <[email protected]>--- fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)
Ping? Omar confirms it fixes xfs for him, so all this needs now is a formal review...
--D
Quoted 163 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the
tr_writeandtr_itruncatelog reservation calculation."The following reproducer results in a transaction log overrun warning for me:
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0
/dev/vdbmount -o rtdev=/dev/vdc/dev/vdb/mnt
fallocate -l 4G/mnt/fooReported-by Omar Sandoval <[email protected]>Signed-off-by Darrick J. Wong <[email protected]>--- fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)fs/xfs/libxfs/xfs_trans_resv.c +77
-19@@ -196,6 +196,24 @@ xfs_calc_inode_chunk_res(return res; } +/* +* Per-extent log reservation for the btree changes involved in freeing or +* allocating a realtime extent. We have to be able to log as many rtbitmap +* blocks as needed to mark inuse MAXEXTLEN blocks' worth of realtime extents, +* as well as the realtime summary block. +*/ +unsigned int +xfs_rtalloc_log_count ( +struct xfs_mount *mp, +unsigned int num_ops) +{ +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); +unsigned int rtbmp_bytes; + + rtbmp_bytes = (MAXEXTLEN / mp->m_sb.sb_rextsize) /NBBY ; +return (howmany (rtbmp_bytes, blksz) +1 ) * num_ops; +} +/* * Various log reservation values. * @@ -218,13 +236,21 @@ xfs_calc_inode_chunk_res(/* * In a write transaction we can allocate a maximum of 2 -* extents. This gives: +* extents. This gives (t1): * the inode getting the new extents: inode size * the inode's bmap btree: max depth * block size * the agfs of the ags from which the extents are allocated: 2 * sector * the superblock free block counter: sector size * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size -* And the bmap_finish transaction can free bmap blocks in a join: +* Or, if we're writing to a realtime file (t2): +* the inode getting the new extents: inode size +* the inode's bmap btree: max depth * block size +* the agfs of the ags from which the extents are allocated: 2 * sector +* the superblock free block counter: sector size +* the realtime bitmap: ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 1 block +* the allocation btrees: 2 trees * (2 * max depth - 1) * block size +* And the bmap_finish transaction can free bmap blocks in a join (t3): * the agfs of the ags containing the blocks: 2 * sector size * the agfls of the ags containing the blocks: 2 * sector size * the super block free block counter: sector size @@ -234,40 +260,72 @@ STATIC uintxfs_calc_write_reservation ( struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), blksz) + +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t2 =xfs_calc_inode_res (mp,1 ) +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), -XFS_FSB_TO_B (mp,1 )) + + blksz) +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 )))); +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,1 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,1 ), blksz); + }else { + t2 =0 ; + } + + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/* -* In truncating a file we free up to two extents at once. We can modify: +* In truncating a file we free up to two extents at once. We can modify (t1): * the inode being truncated: inode size * the inode's bmap btree: (max depth + 1) * block size -* And the bmap_finish transaction can free the blocks and bmap blocks: +* And the bmap_finish transaction can free the blocks and bmap blocks (t2): * the agf for each of the ags: 4 * sector size * the agfl for each of the ags: 4 * sector size * the super block to reflect the freed blocks: sector size * worst case split in allocation btrees per extent assuming 4 extents: * 4 exts * 2 trees * (2 * max depth - 1) * block size +* Or, if it's a realtime file (t3): +* the agf for each of the ags: 2 * sector size +* the agfl for each of the ags: 2 * sector size +* the super block to reflect the freed blocks: sector size +* the realtime bitmap: 2 exts * ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 2 exts * 1 block +* worst case split in allocation btrees per extent assuming 2 extents: +* 2 exts * 2 trees * (2 * max depth - 1) * block size */ STATIC uintxfs_calc_itruncate_reservation (struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + -xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), -XFS_FSB_TO_B (mp,1 )))); +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , blksz); + + t2 =xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,2 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + }else { + t3 =0 ; + } + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/*
Quoted 10 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the
tr_writeandtr_itruncatelog reservation calculation."The following reproducer results in a transaction log overrun warning for me:
Quoted 7 lines
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0
/dev/vdbmount -o rtdev=/dev/vdc/dev/vdb/mnt
fallocate -l 4G/mnt/fooReported-by Omar Sandoval <[email protected]>Signed-off-by Darrick J. Wong <[email protected]>---
Looks reasonable enough given my limited knowledge on the rt bits. One question..
Quoted 42 lines
fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)fs/xfs/libxfs/xfs_trans_resv.c +19
-1@@ -196,6 +196,24 @@ xfs_calc_inode_chunk_res(return res; } +/* +* Per-extent log reservation for the btree changes involved in freeing or +* allocating a realtime extent. We have to be able to log as many rtbitmap +* blocks as needed to mark inuse MAXEXTLEN blocks' worth of realtime extents, +* as well as the realtime summary block. +*/ +unsigned int +xfs_rtalloc_log_count ( +struct xfs_mount *mp, +unsigned int num_ops) +{ +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); +unsigned int rtbmp_bytes; + + rtbmp_bytes = (MAXEXTLEN / mp->m_sb.sb_rextsize) /NBBY ; +return (howmany (rtbmp_bytes, blksz) +1 ) * num_ops; +} +/* * Various log reservation values. * @@ -218,13 +236,21 @@ xfs_calc_inode_chunk_res(/* * In a write transaction we can allocate a maximum of 2 -* extents. This gives: +* extents. This gives (t1): * the inode getting the new extents: inode size * the inode's bmap btree: max depth * block size * the agfs of the ags from which the extents are allocated: 2 * sector * the superblock free block counter: sector size
fs/xfs/libxfs/xfs_trans_resv.c xfs_calc_inode_chunk_res( · line 252
Quoted 10 lines
fs/xfs/libxfs/xfs_trans_resv.c +8
-1* the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size -* And the bmap_finish transaction can free bmap blocks in a join: +* Or, if we're writing to a realtime file (t2): +* the inode getting the new extents: inode size +* the inode's bmap btree: max depth * block size +* the agfs of the ags from which the extents are allocated: 2 * sector +* the superblock free block counter: sector size +* the realtime bitmap: ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 1 block +* the allocation btrees: 2 trees * (2 * max depth - 1) * block size
Why do we include the allocation btrees in the rt reservations? I thought that we'd either allocate (or free) out of one pool or the other. Do we operate on both sets of structures in the same transaction?
Brian
Quoted 95 lines
fs/xfs/libxfs/xfs_trans_resv.c +50
-17+* And the bmap_finish transaction can free bmap blocks in a join (t3): * the agfs of the ags containing the blocks: 2 * sector size * the agfls of the ags containing the blocks: 2 * sector size * the super block free block counter: sector size @@ -234,40 +260,72 @@ STATIC uintxfs_calc_write_reservation ( struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), blksz) + +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t2 =xfs_calc_inode_res (mp,1 ) +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), -XFS_FSB_TO_B (mp,1 )) + + blksz) +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 )))); +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,1 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,1 ), blksz); + }else { + t2 =0 ; + } + + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/* -* In truncating a file we free up to two extents at once. We can modify: +* In truncating a file we free up to two extents at once. We can modify (t1): * the inode being truncated: inode size * the inode's bmap btree: (max depth + 1) * block size -* And the bmap_finish transaction can free the blocks and bmap blocks: +* And the bmap_finish transaction can free the blocks and bmap blocks (t2): * the agf for each of the ags: 4 * sector size * the agfl for each of the ags: 4 * sector size * the super block to reflect the freed blocks: sector size * worst case split in allocation btrees per extent assuming 4 extents: * 4 exts * 2 trees * (2 * max depth - 1) * block size +* Or, if it's a realtime file (t3): +* the agf for each of the ags: 2 * sector size +* the agfl for each of the ags: 2 * sector size +* the super block to reflect the freed blocks: sector size +* the realtime bitmap: 2 exts * ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 2 exts * 1 block +* worst case split in allocation btrees per extent assuming 2 extents: +* 2 exts * 2 trees * (2 * max depth - 1) * block size */ STATIC uintxfs_calc_itruncate_reservation (struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + -xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), -XFS_FSB_TO_B (mp,1 )))); +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , blksz); + + t2 =xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,2 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + }else { + t3 =0 ; + } + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/*
Quoted 70 lines · Brian Foster
Quoted 10 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the
tr_writeandtr_itruncatelog reservation calculation."The following reproducer results in a transaction log overrun warning for me:
Quoted 7 lines
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0
/dev/vdbmount -o rtdev=/dev/vdc/dev/vdb/mnt
fallocate -l 4G/mnt/fooReported-by Omar Sandoval <[email protected]>Signed-off-by Darrick J. Wong <[email protected]>---
Looks reasonable enough given my limited knowledge on the rt bits. One question..
Quoted 48 lines
fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)fs/xfs/libxfs/xfs_trans_resv.c +23
-2@@ -196,6 +196,24 @@ xfs_calc_inode_chunk_res(return res; } +/* +* Per-extent log reservation for the btree changes involved in freeing or +* allocating a realtime extent. We have to be able to log as many rtbitmap +* blocks as needed to mark inuse MAXEXTLEN blocks' worth of realtime extents, +* as well as the realtime summary block. +*/ +unsigned int +xfs_rtalloc_log_count ( +struct xfs_mount *mp, +unsigned int num_ops) +{ +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); +unsigned int rtbmp_bytes; + + rtbmp_bytes = (MAXEXTLEN / mp->m_sb.sb_rextsize) /NBBY ; +return (howmany (rtbmp_bytes, blksz) +1 ) * num_ops; +} +/* * Various log reservation values. * @@ -218,13 +236,21 @@ xfs_calc_inode_chunk_res(/* * In a write transaction we can allocate a maximum of 2 -* extents. This gives: +* extents. This gives (t1): * the inode getting the new extents: inode size * the inode's bmap btree: max depth * block size * the agfs of the ags from which the extents are allocated: 2 * sector * the superblock free block counter: sector size * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size -* And the bmap_finish transaction can free bmap blocks in a join: +* Or, if we're writing to a realtime file (t2): +* the inode getting the new extents: inode size +* the inode's bmap btree: max depth * block size +* the agfs of the ags from which the extents are allocated: 2 * sector
Quoted 8 lines
fs/xfs/libxfs/xfs_trans_resv.c xfs_calc_inode_chunk_res( · line 252
Quoted 4 lines
fs/xfs/libxfs/xfs_trans_resv.c +4
-0+* the superblock free block counter: sector size +* the realtime bitmap: ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 1 block +* the allocation btrees: 2 trees * (2 * max depth - 1) * block size Why do we include the allocation btrees in the rt reservations? I thought that we'd either allocate (or free) out of one pool or the other. Do we operate on both sets of structures in the same transaction?
I read "allocation btrees: 2 exts * 2 trees..." for t1 to mean that we need to be able to allocate one datadev extent (which could cause a full bnobt/cntbt split) for the actual file data, and then the second extent is to handle allocating a new bmbt block to the bmap btree.
Based on that, I concluded that we still need to reserve space for that "second" extent to handle allocating a new bmbt block (on the datadev).
While pondering that, I wondered if even that's really true because what happens if you suffer a full bmbt split, the free space is so fragmented that each level of the bmbt split ends up allocating a new block from a different part of the free space btrees and that in turn causes splits in the free space btrees?
I think the answer is "no we're fine" because even if each new bmbt block comes from a different bnobt record, a full bmbt split will never hollow out more than half of one bnobt block's worth of free space records.
--D
Quoted 98 lines
Brian
Quoted 95 lines
fs/xfs/libxfs/xfs_trans_resv.c +50
-17+* And the bmap_finish transaction can free bmap blocks in a join (t3): * the agfs of the ags containing the blocks: 2 * sector size * the agfls of the ags containing the blocks: 2 * sector size * the super block free block counter: sector size @@ -234,40 +260,72 @@ STATIC uintxfs_calc_write_reservation ( struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), blksz) + +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t2 =xfs_calc_inode_res (mp,1 ) +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), -XFS_FSB_TO_B (mp,1 )) + + blksz) +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 )))); +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,1 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,1 ), blksz); + }else { + t2 =0 ; + } + + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/* -* In truncating a file we free up to two extents at once. We can modify: +* In truncating a file we free up to two extents at once. We can modify (t1): * the inode being truncated: inode size * the inode's bmap btree: (max depth + 1) * block size -* And the bmap_finish transaction can free the blocks and bmap blocks: +* And the bmap_finish transaction can free the blocks and bmap blocks (t2): * the agf for each of the ags: 4 * sector size * the agfl for each of the ags: 4 * sector size * the super block to reflect the freed blocks: sector size * worst case split in allocation btrees per extent assuming 4 extents: * 4 exts * 2 trees * (2 * max depth - 1) * block size +* Or, if it's a realtime file (t3): +* the agf for each of the ags: 2 * sector size +* the agfl for each of the ags: 2 * sector size +* the super block to reflect the freed blocks: sector size +* the realtime bitmap: 2 exts * ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 2 exts * 1 block +* worst case split in allocation btrees per extent assuming 2 extents: +* 2 exts * 2 trees * (2 * max depth - 1) * block size */ STATIC uintxfs_calc_itruncate_reservation (struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + -xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), -XFS_FSB_TO_B (mp,1 )))); +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , blksz); + + t2 =xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,2 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + }else { + t3 =0 ; + } + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/*
Quoted 80 lines · Darrick J. Wong
Quoted 70 lines · Brian Foster
Quoted 10 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the
tr_writeandtr_itruncatelog reservation calculation."The following reproducer results in a transaction log overrun warning for me:
Quoted 7 lines
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0
/dev/vdbmount -o rtdev=/dev/vdc/dev/vdb/mnt
fallocate -l 4G/mnt/fooReported-by Omar Sandoval <[email protected]>Signed-off-by Darrick J. Wong <[email protected]>---
Looks reasonable enough given my limited knowledge on the rt bits. One question..
Quoted 48 lines
fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)fs/xfs/libxfs/xfs_trans_resv.c +23
-2@@ -196,6 +196,24 @@ xfs_calc_inode_chunk_res(return res; } +/* +* Per-extent log reservation for the btree changes involved in freeing or +* allocating a realtime extent. We have to be able to log as many rtbitmap +* blocks as needed to mark inuse MAXEXTLEN blocks' worth of realtime extents, +* as well as the realtime summary block. +*/ +unsigned int +xfs_rtalloc_log_count ( +struct xfs_mount *mp, +unsigned int num_ops) +{ +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); +unsigned int rtbmp_bytes; + + rtbmp_bytes = (MAXEXTLEN / mp->m_sb.sb_rextsize) /NBBY ; +return (howmany (rtbmp_bytes, blksz) +1 ) * num_ops; +} +/* * Various log reservation values. * @@ -218,13 +236,21 @@ xfs_calc_inode_chunk_res(/* * In a write transaction we can allocate a maximum of 2 -* extents. This gives: +* extents. This gives (t1): * the inode getting the new extents: inode size * the inode's bmap btree: max depth * block size * the agfs of the ags from which the extents are allocated: 2 * sector * the superblock free block counter: sector size * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size -* And the bmap_finish transaction can free bmap blocks in a join: +* Or, if we're writing to a realtime file (t2): +* the inode getting the new extents: inode size +* the inode's bmap btree: max depth * block size +* the agfs of the ags from which the extents are allocated: 2 * sector Quoted 8 lines
fs/xfs/libxfs/xfs_trans_resv.c xfs_calc_inode_chunk_res( · line 252
Quoted 4 lines
fs/xfs/libxfs/xfs_trans_resv.c +4
-0+* the superblock free block counter: sector size +* the realtime bitmap: ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 1 block +* the allocation btrees: 2 trees * (2 * max depth - 1) * block size Why do we include the allocation btrees in the rt reservations? I thought that we'd either allocate (or free) out of one pool or the other. Do we operate on both sets of structures in the same transaction?
Quoted 5 lines
I read "allocation btrees: 2 exts * 2 trees..." for t1 to mean that we need to be able to allocate one datadev extent (which could cause a full bnobt/cntbt split) for the actual file data, and then the second extent is to handle allocating a new bmbt block to the bmap btree.
Ah, metadata out of the traditional trees.. that makes sense. My general understanding is that we have two sets of free space and thus two associated free space tracking structures: the traditional perag btrees for the local device and some bitmap indexing scheme for the external realtime device. Based on that, it looks like a file data allocation falls down into xfs_bmap_rtalloc() to allocate data blocks via the RT subsystem and the subsequent bmap update falls into the bmapbt code that uses xfs_alloc_vextent() directly to allocate blocks for the bmbt.
With regard to the 2 extents, the first sentence in the comment above suggests to me that the two extents is a per-transaction operational limit. IOW, a write transaction supports two xfs_bmapi_write() calls, for example, as opposed to referring to the two lower level allocations outlined above. That seems consistent with the "2 * sector" AGF portion of the reservation as well, but I could easily be wrong about that.
BTW I'm not following what you mean by a datadev extent causing a bnobt/cntbt split. Doesn't the data extent come from the RT free space, or are you just indirectly referring to the supporting bmbt block allocation causing a split..?
Quoted 9 lines
Based on that, I concluded that we still need to reserve space for that "second" extent to handle allocating a new bmbt block (on the datadev).
While pondering that, I wondered if even that's really true because what happens if you suffer a full bmbt split, the free space is so fragmented that each level of the bmbt split ends up allocating a new block from a different part of the free space btrees and that in turn causes splits in the free space btrees?
Quoted 5 lines
I think the answer is "no we're fine" because even if each new bmbt block comes from a different bnobt record, a full bmbt split will never hollow out more than half of one bnobt block's worth of free space records.
Yeah. I couldn't say for sure, but this strikes me as something we'd see reports of if it were possible/likely to occur in practice, particularly since this is the basis of how the allocation transaction reservations are calculated in general (not just for RT).
Brian
Quoted 101 lines
--D
Quoted 98 lines
Brian
Quoted 95 lines
fs/xfs/libxfs/xfs_trans_resv.c +50
-17+* And the bmap_finish transaction can free bmap blocks in a join (t3): * the agfs of the ags containing the blocks: 2 * sector size * the agfls of the ags containing the blocks: 2 * sector size * the super block free block counter: sector size @@ -234,40 +260,72 @@ STATIC uintxfs_calc_write_reservation ( struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), blksz) + +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t2 =xfs_calc_inode_res (mp,1 ) +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), -XFS_FSB_TO_B (mp,1 )) + + blksz) +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 )))); +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,1 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,1 ), blksz); + }else { + t2 =0 ; + } + + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/* -* In truncating a file we free up to two extents at once. We can modify: +* In truncating a file we free up to two extents at once. We can modify (t1): * the inode being truncated: inode size * the inode's bmap btree: (max depth + 1) * block size -* And the bmap_finish transaction can free the blocks and bmap blocks: +* And the bmap_finish transaction can free the blocks and bmap blocks (t2): * the agf for each of the ags: 4 * sector size * the agfl for each of the ags: 4 * sector size * the super block to reflect the freed blocks: sector size * worst case split in allocation btrees per extent assuming 4 extents: * 4 exts * 2 trees * (2 * max depth - 1) * block size +* Or, if it's a realtime file (t3): +* the agf for each of the ags: 2 * sector size +* the agfl for each of the ags: 2 * sector size +* the super block to reflect the freed blocks: sector size +* the realtime bitmap: 2 exts * ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 2 exts * 1 block +* worst case split in allocation btrees per extent assuming 2 extents: +* 2 exts * 2 trees * (2 * max depth - 1) * block size */ STATIC uintxfs_calc_itruncate_reservation (struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + -xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), -XFS_FSB_TO_B (mp,1 )))); +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , blksz); + + t2 =xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,2 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + }else { + t3 =0 ; + } + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/*
Quoted 103 lines · Brian Foster
Quoted 80 lines · Darrick J. Wong
Quoted 70 lines · Brian Foster
Quoted 10 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the
tr_writeandtr_itruncatelog reservation calculation."The following reproducer results in a transaction log overrun warning for me:
Quoted 7 lines
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0
/dev/vdbmount -o rtdev=/dev/vdc/dev/vdb/mnt
fallocate -l 4G/mnt/fooReported-by Omar Sandoval <[email protected]>Signed-off-by Darrick J. Wong <[email protected]>---
Looks reasonable enough given my limited knowledge on the rt bits. One question..
Quoted 48 lines
fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)fs/xfs/libxfs/xfs_trans_resv.c +23
-2@@ -196,6 +196,24 @@ xfs_calc_inode_chunk_res(return res; } +/* +* Per-extent log reservation for the btree changes involved in freeing or +* allocating a realtime extent. We have to be able to log as many rtbitmap +* blocks as needed to mark inuse MAXEXTLEN blocks' worth of realtime extents, +* as well as the realtime summary block. +*/ +unsigned int +xfs_rtalloc_log_count ( +struct xfs_mount *mp, +unsigned int num_ops) +{ +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); +unsigned int rtbmp_bytes; + + rtbmp_bytes = (MAXEXTLEN / mp->m_sb.sb_rextsize) /NBBY ; +return (howmany (rtbmp_bytes, blksz) +1 ) * num_ops; +} +/* * Various log reservation values. * @@ -218,13 +236,21 @@ xfs_calc_inode_chunk_res(/* * In a write transaction we can allocate a maximum of 2 -* extents. This gives: +* extents. This gives (t1): * the inode getting the new extents: inode size * the inode's bmap btree: max depth * block size * the agfs of the ags from which the extents are allocated: 2 * sector * the superblock free block counter: sector size * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size -* And the bmap_finish transaction can free bmap blocks in a join: +* Or, if we're writing to a realtime file (t2): +* the inode getting the new extents: inode size +* the inode's bmap btree: max depth * block size +* the agfs of the ags from which the extents are allocated: 2 * sector Quoted 8 lines
fs/xfs/libxfs/xfs_trans_resv.c xfs_calc_inode_chunk_res( · line 252
Quoted 4 lines
fs/xfs/libxfs/xfs_trans_resv.c +4
-0+* the superblock free block counter: sector size +* the realtime bitmap: ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 1 block +* the allocation btrees: 2 trees * (2 * max depth - 1) * block size Why do we include the allocation btrees in the rt reservations? I thought that we'd either allocate (or free) out of one pool or the other. Do we operate on both sets of structures in the same transaction?
Quoted 5 lines
I read "allocation btrees: 2 exts * 2 trees..." for t1 to mean that we need to be able to allocate one datadev extent (which could cause a full bnobt/cntbt split) for the actual file data, and then the second extent is to handle allocating a new bmbt block to the bmap btree.
Ah, metadata out of the traditional trees.. that makes sense. My general understanding is that we have two sets of free space and thus two associated free space tracking structures: the traditional perag btrees for the local device and some bitmap indexing scheme for the external realtime device. Based on that, it looks like a file data allocation falls down into
xfs_bmap_rtalloc()to allocate data blocks via the RT subsystem and the subsequent bmap update falls into the bmapbt code that usesxfs_alloc_vextent()directly to allocate blocks for the bmbt.With regard to the 2 extents, the first sentence in the comment above suggests to me that the two extents is a per-transaction operational limit. IOW, a write transaction supports two
xfs_bmapi_write()calls, for example, as opposed to referring to the two lower level allocations outlined above. That seems consistent with the "2 * sector" AGF portion of the reservation as well, but I could easily be wrong about that.
Quoted 4 lines
BTW I'm not following what you mean by a datadev extent causing a bnobt/cntbt split. Doesn't the data extent come from the RT free space, or are you just indirectly referring to the supporting bmbt block allocation causing a split..?
"t1" is reflects writes to regular files on the datadev device, so "allocate one datadev extent (which could cause a full bnobt/cntbt split) for actual file data" applies to that case, not t2.
"t2" is for writes to realtime files on the rt device.
Quoted 2 lines
Quoted 2 lines
Based on that, I concluded that we still need to reserve space for that "second" extent to handle allocating a new bmbt block (on the datadev).
Perhaps my reply could have been clearer had I said:
"Based on that, I concluded for the realtime case (t2) that we still..."
Quoted 12 lines
Quoted 11 lines
While pondering that, I wondered if even that's really true because what happens if you suffer a full bmbt split, the free space is so fragmented that each level of the bmbt split ends up allocating a new block from a different part of the free space btrees and that in turn causes splits in the free space btrees?
I think the answer is "no we're fine" because even if each new bmbt block comes from a different bnobt record, a full bmbt split will never hollow out more than half of one bnobt block's worth of free space records.
Quoted 4 lines
Yeah. I couldn't say for sure, but this strikes me as something we'd see reports of if it were possible/likely to occur in practice, particularly since this is the basis of how the allocation transaction reservations are calculated in general (not just for RT).
<nod>
--D
Quoted 104 lines
Brian
Quoted 101 lines
--D
Quoted 98 lines
Brian
Quoted 95 lines
fs/xfs/libxfs/xfs_trans_resv.c +50
-17+* And the bmap_finish transaction can free bmap blocks in a join (t3): * the agfs of the ags containing the blocks: 2 * sector size * the agfls of the ags containing the blocks: 2 * sector size * the super block free block counter: sector size @@ -234,40 +260,72 @@ STATIC uintxfs_calc_write_reservation ( struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), blksz) + +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t2 =xfs_calc_inode_res (mp,1 ) +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), -XFS_FSB_TO_B (mp,1 )) + + blksz) +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 )))); +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,1 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,1 ), blksz); + }else { + t2 =0 ; + } + + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/* -* In truncating a file we free up to two extents at once. We can modify: +* In truncating a file we free up to two extents at once. We can modify (t1): * the inode being truncated: inode size * the inode's bmap btree: (max depth + 1) * block size -* And the bmap_finish transaction can free the blocks and bmap blocks: +* And the bmap_finish transaction can free the blocks and bmap blocks (t2): * the agf for each of the ags: 4 * sector size * the agfl for each of the ags: 4 * sector size * the super block to reflect the freed blocks: sector size * worst case split in allocation btrees per extent assuming 4 extents: * 4 exts * 2 trees * (2 * max depth - 1) * block size +* Or, if it's a realtime file (t3): +* the agf for each of the ags: 2 * sector size +* the agfl for each of the ags: 2 * sector size +* the super block to reflect the freed blocks: sector size +* the realtime bitmap: 2 exts * ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 2 exts * 1 block +* worst case split in allocation btrees per extent assuming 2 extents: +* 2 exts * 2 trees * (2 * max depth - 1) * block size */ STATIC uintxfs_calc_itruncate_reservation (struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + -xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), -XFS_FSB_TO_B (mp,1 )))); +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , blksz); + + t2 =xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,2 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + }else { + t3 =0 ; + } + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/*
Brian Foster
Reviewed-byQuoted 113 lines · Darrick J. Wong
Quoted 103 lines · Brian Foster
Quoted 80 lines · Darrick J. Wong
Quoted 70 lines · Brian Foster
Quoted 10 lines · Darrick J. Wong
From: Darrick J. Wong <[email protected]>
Omar Sandoval reported that a 4G fallocate on the realtime device causes filesystem shutdowns due to a log reservation overflow that happens when we log the rtbitmap updates. Factor rtbitmap/rtsummary updates into the the
tr_writeandtr_itruncatelog reservation calculation."The following reproducer results in a transaction log overrun warning for me:
Quoted 7 lines
mkfs.xfs -f -r rtdev=/dev/vdc -d rtinherit=1 -m reflink=0
/dev/vdbmount -o rtdev=/dev/vdc/dev/vdb/mnt
fallocate -l 4G/mnt/fooReported-by Omar Sandoval <[email protected]>Signed-off-by Darrick J. Wong <[email protected]>---
Looks reasonable enough given my limited knowledge on the rt bits. One question..
Quoted 48 lines
fs/xfs/libxfs/xfs_trans_resv.c | 96 ++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-)fs/xfs/libxfs/xfs_trans_resv.c +23
-2@@ -196,6 +196,24 @@ xfs_calc_inode_chunk_res(return res; } +/* +* Per-extent log reservation for the btree changes involved in freeing or +* allocating a realtime extent. We have to be able to log as many rtbitmap +* blocks as needed to mark inuse MAXEXTLEN blocks' worth of realtime extents, +* as well as the realtime summary block. +*/ +unsigned int +xfs_rtalloc_log_count ( +struct xfs_mount *mp, +unsigned int num_ops) +{ +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); +unsigned int rtbmp_bytes; + + rtbmp_bytes = (MAXEXTLEN / mp->m_sb.sb_rextsize) /NBBY ; +return (howmany (rtbmp_bytes, blksz) +1 ) * num_ops; +} +/* * Various log reservation values. * @@ -218,13 +236,21 @@ xfs_calc_inode_chunk_res(/* * In a write transaction we can allocate a maximum of 2 -* extents. This gives: +* extents. This gives (t1): * the inode getting the new extents: inode size * the inode's bmap btree: max depth * block size * the agfs of the ags from which the extents are allocated: 2 * sector * the superblock free block counter: sector size * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size -* And the bmap_finish transaction can free bmap blocks in a join: +* Or, if we're writing to a realtime file (t2): +* the inode getting the new extents: inode size +* the inode's bmap btree: max depth * block size +* the agfs of the ags from which the extents are allocated: 2 * sector Quoted 8 lines
fs/xfs/libxfs/xfs_trans_resv.c xfs_calc_inode_chunk_res( · line 252
Quoted 4 lines
fs/xfs/libxfs/xfs_trans_resv.c +4
-0+* the superblock free block counter: sector size +* the realtime bitmap: ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 1 block +* the allocation btrees: 2 trees * (2 * max depth - 1) * block size Why do we include the allocation btrees in the rt reservations? I thought that we'd either allocate (or free) out of one pool or the other. Do we operate on both sets of structures in the same transaction?
Quoted 5 lines
I read "allocation btrees: 2 exts * 2 trees..." for t1 to mean that we need to be able to allocate one datadev extent (which could cause a full bnobt/cntbt split) for the actual file data, and then the second extent is to handle allocating a new bmbt block to the bmap btree.
Ah, metadata out of the traditional trees.. that makes sense. My general understanding is that we have two sets of free space and thus two associated free space tracking structures: the traditional perag btrees for the local device and some bitmap indexing scheme for the external realtime device. Based on that, it looks like a file data allocation falls down into
xfs_bmap_rtalloc()to allocate data blocks via the RT subsystem and the subsequent bmap update falls into the bmapbt code that usesxfs_alloc_vextent()directly to allocate blocks for the bmbt.With regard to the 2 extents, the first sentence in the comment above suggests to me that the two extents is a per-transaction operational limit. IOW, a write transaction supports two
xfs_bmapi_write()calls, for example, as opposed to referring to the two lower level allocations outlined above. That seems consistent with the "2 * sector" AGF portion of the reservation as well, but I could easily be wrong about that.Quoted 4 lines
BTW I'm not following what you mean by a datadev extent causing a bnobt/cntbt split. Doesn't the data extent come from the RT free space, or are you just indirectly referring to the supporting bmbt block allocation causing a split..?
"t1" is reflects writes to regular files on the datadev device, so "allocate one datadev extent (which could cause a full bnobt/cntbt split) for actual file data" applies to that case, not t2.
Quoted 9 lines
"t2" is for writes to realtime files on the rt device.
Quoted 2 lines
Quoted 2 lines
Based on that, I concluded that we still need to reserve space for that "second" extent to handle allocating a new bmbt block (on the datadev).
Perhaps my reply could have been clearer had I said:
"Based on that, I concluded for the realtime case (t2) that we still..."
Ok. In any event, patch seems reasonable to me:
Quoted 126 lines
Quoted 12 lines
Quoted 11 lines
While pondering that, I wondered if even that's really true because what happens if you suffer a full bmbt split, the free space is so fragmented that each level of the bmbt split ends up allocating a new block from a different part of the free space btrees and that in turn causes splits in the free space btrees?
I think the answer is "no we're fine" because even if each new bmbt block comes from a different bnobt record, a full bmbt split will never hollow out more than half of one bnobt block's worth of free space records.
Quoted 4 lines
Yeah. I couldn't say for sure, but this strikes me as something we'd see reports of if it were possible/likely to occur in practice, particularly since this is the basis of how the allocation transaction reservations are calculated in general (not just for RT).
<nod>
--D
Quoted 104 lines
Brian
Quoted 101 lines
--D
Quoted 98 lines
Brian
Quoted 95 lines
fs/xfs/libxfs/xfs_trans_resv.c +50
-17+* And the bmap_finish transaction can free bmap blocks in a join (t3): * the agfs of the ags containing the blocks: 2 * sector size * the agfls of the ags containing the blocks: 2 * sector size * the super block free block counter: sector size @@ -234,40 +260,72 @@ STATIC uintxfs_calc_write_reservation ( struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), blksz) + +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t2 =xfs_calc_inode_res (mp,1 ) +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ), -XFS_FSB_TO_B (mp,1 )) + + blksz) +xfs_calc_buf_res (3 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), -XFS_FSB_TO_B (mp,1 )))); +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,1 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,1 ), blksz); + }else { + t2 =0 ; + } + + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/* -* In truncating a file we free up to two extents at once. We can modify: +* In truncating a file we free up to two extents at once. We can modify (t1): * the inode being truncated: inode size * the inode's bmap btree: (max depth + 1) * block size -* And the bmap_finish transaction can free the blocks and bmap blocks: +* And the bmap_finish transaction can free the blocks and bmap blocks (t2): * the agf for each of the ags: 4 * sector size * the agfl for each of the ags: 4 * sector size * the super block to reflect the freed blocks: sector size * worst case split in allocation btrees per extent assuming 4 extents: * 4 exts * 2 trees * (2 * max depth - 1) * block size +* Or, if it's a realtime file (t3): +* the agf for each of the ags: 2 * sector size +* the agfl for each of the ags: 2 * sector size +* the super block to reflect the freed blocks: sector size +* the realtime bitmap: 2 exts * ((MAXEXTLEN / rtextsize) / NBBY) bytes +* the realtime summary: 2 exts * 1 block +* worst case split in allocation btrees per extent assuming 2 extents: +* 2 exts * 2 trees * (2 * max depth - 1) * block size */ STATIC uintxfs_calc_itruncate_reservation (struct xfs_mount *mp) { -return XFS_DQUOT_LOGRES (mp) + -max ((xfs_calc_inode_res (mp,1 ) + -xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , -XFS_FSB_TO_B (mp,1 ))), - (xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + -xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), -XFS_FSB_TO_B (mp,1 )))); +unsigned int t1, t2, t3; +unsigned int blksz =XFS_FSB_TO_B (mp,1 ); + + t1 =xfs_calc_inode_res (mp,1 ) + +xfs_calc_buf_res (XFS_BM_MAXLEVELS (mp,XFS_DATA_FORK ) +1 , blksz); + + t2 =xfs_calc_buf_res (9 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,4 ), blksz); + +if (xfs_sb_version_hasrealtime (&mp->m_sb)) { + t3 =xfs_calc_buf_res (5 , mp->m_sb.sb_sectsize) + +xfs_calc_buf_res (xfs_rtalloc_log_count (mp,2 ), blksz) + +xfs_calc_buf_res (xfs_allocfree_log_count (mp,2 ), blksz); + }else { + t3 =0 ; + } + +return XFS_DQUOT_LOGRES (mp) +max3 (t1, t2, t3); }/*