Add agskip=value mount option

The agskip mount option specifies the allocation group, (AG) for a new file, relative to the start of the last created file. agskip has the
opposite effect of the rotorstep system tunable parameter. Each
new file to be placed in the location lastAG + agskipValue, where lastAG is the allocation group of the last created file.

For example, agskip=3 means each new file will be allocated three AGs away from the starting AG of the most recently created file.

The agskip mount option disables the rotorstep system tunable parameter.

Signed-off-by Rich Johnston <[email protected]>

---
 fs/xfs/xfs_alloc.c      |   35 ++++++++++++++++++++++++++++-------
 fs/xfs/xfs_filestream.c |    8 +++++++-
 fs/xfs/xfs_mount.c      |    1 +
 fs/xfs/xfs_mount.h      |    4 +++-
 fs/xfs/xfs_super.c      |   13 ++++++++++++-
 5 files changed, 51 insertions(+), 10 deletions(-)

Index: b/fs/xfs/xfs_alloc.c
===================================================================

fs/xfs/xfs_alloc.c +0 -0

@@ -2365,12 +2365,20 @@ xfs_alloc_vextent(
 		 * Try near allocation first, then anywhere-in-ag after
 		 * the first a.g. fails.

fs/xfs/xfs_alloc.c +11 -0

+				args->fsbno = XFS_AGB_TO_FSB(mp,
+					mp->m_agfrotor, 0);
+				mp->m_agfrotor = (mp->m_agfrotor + mp->m_agskip)
+					% mp->m_sb.sb_agcount;
+				spin_unlock(&mp->m_agfrotor_lock);
+			} else if (mp->m_flags & XFS_MOUNT_32BITINODES) {
+				args->fsbno = XFS_AGB_TO_FSB(mp,
+						((mp->m_agfrotor / rotorstep) %
+						mp->m_sb.sb_agcount), 0);
+				bump_rotor = 1;
+			}
 		}
 		args->agbno = XFS_FSB_TO_AGBNO(mp, args->fsbno);
 		args->type = XFS_ALLOCTYPE_NEAR_BNO;

fs/xfs/xfs_alloc.c +4 -0

+                        } else {
+				sagno = (mp->m_agfrotor / rotorstep) %
 					mp->m_sb.sb_agcount;
+			}
+			args->agno = sagno;
 			args->type = XFS_ALLOCTYPE_THIS_AG;
 			flags = XFS_ALLOC_FLAG_TRYLOCK;
 		} else if (type == XFS_ALLOCTYPE_FIRST_AG) {

Index: b/fs/xfs/xfs_filestream.c
===================================================================

fs/xfs/xfs_filestream.c +7 -1

@@ -622,7 +622,13 @@ xfs_filestream_associate(
 	 * Set the starting AG using the rotor for inode32, otherwise
 	 * use the directory inode's AG.
 	 */
-	if (mp->m_flags & XFS_MOUNT_32BITINODES) {
+	if (mp->m_flags & XFS_MOUNT_AGSKIP) {
+		spin_lock(&mp->m_agfrotor_lock);
+		startag = mp->m_agfrotor;
+		mp->m_agfrotor = (mp->m_agfrotor + mp->m_agskip)
+			% mp->m_sb.sb_agcount;
+		spin_unlock(&mp->m_agfrotor_lock);
+	} else if (mp->m_flags & XFS_MOUNT_32BITINODES) {
 		rotorstep = xfs_rotorstep;
 		startag = (mp->m_agfrotor / rotorstep) % mp->m_sb.sb_agcount;
 		mp->m_agfrotor = (mp->m_agfrotor + 1) %

Index: b/fs/xfs/xfs_mount.c
===================================================================

fs/xfs/xfs_mount.c +1 -0

@@ -768,6 +768,7 @@ STATIC void
 xfs_mount_common(xfs_mount_t *mp, xfs_sb_t *sbp)
 {
 	mp->m_agfrotor = mp->m_agirotor = 0;
+	spin_lock_init(&mp->m_agfrotor_lock);
 	spin_lock_init(&mp->m_agirotor_lock);
 	mp->m_maxagi = mp->m_sb.sb_agcount;
 	mp->m_blkbit_log = sbp->sb_blocklog + XFS_NBBYLOG;

Index: b/fs/xfs/xfs_mount.h
===================================================================

fs/xfs/xfs_mount.h +1 -0

@@ -119,6 +119,7 @@ typedef struct xfs_mount {
 	char			*m_logname;	/* external log device name */
 	int			m_bsize;	/* fs logical block size */
 	xfs_agnumber_t		m_agfrotor;	/* last ag where space found */
+	spinlock_t		m_agfrotor_lock;/* .. and lock protecting it */
 	xfs_agnumber_t		m_agirotor;	/* last ag dir inode alloced */
 	spinlock_t		m_agirotor_lock;/* .. and lock protecting it */
 	xfs_agnumber_t		m_maxagi;	/* highest inode alloc group */

fs/xfs/xfs_mount.h +1 -1

 	struct workqueue_struct	*m_data_workqueue;
 	struct workqueue_struct	*m_unwritten_workqueue;

@@ -246,7 +248,7 @@ typedef struct xfs_mount {
 #define XFS_MOUNT_FILESTREAMS	(1ULL << 24)	/* enable the filestreams
 						   allocator */
 #define XFS_MOUNT_NOATTR2	(1ULL << 25)	/* disable use of attr2 format */
-
+#define XFS_MOUNT_AGSKIP	(1ULL << 26)	/* extent allocation stride */
 
 /*
  * Default minimum read and write sizes.

Index: b/fs/xfs/xfs_super.c
===================================================================

fs/xfs/xfs_super.c +11 -0

 /*
  * Table driven mount option parser.
  *

@@ -383,6 +383,15 @@ xfs_parseargs(
 		} else if (!strcmp(this_char, "irixsgid")) {
 			xfs_warn(mp,
 	"irixsgid is now a sysctl(2) variable, option is deprecated.");
+		} else if (!strcmp(this_char, MNTOPT_AGSKIP)) {
+			if (!value || !*value) {
+				xfs_warn(mp,
+					"%s option requires an argument",
+					this_char);
+				return EINVAL;
+			}
+			mp->m_flags |= XFS_MOUNT_AGSKIP;
+			mp->m_agskip = simple_strtoul(value, &eov, 10);
 		} else {
 			xfs_warn(mp, "unknown mount option [%s].", this_char);
 			return EINVAL;

@@ -567,6 +576,8 @@ xfs_showargs(
 
 	if (!(mp->m_qflags & XFS_ALL_QUOTA_ACCT))
 		seq_puts(m, "," MNTOPT_NOQUOTA);
+	if (mp->m_flags & XFS_MOUNT_AGSKIP)
+		seq_printf(m, "," MNTOPT_AGSKIP "=%d", mp->m_agskip);
 
 	return 0;
 }

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

The agskip mount option specifies the allocation group, (AG) for a new file, relative to the start of the last created file. agskip has the
opposite effect of the rotorstep system tunable parameter. Each
new file to be placed in the location lastAG + agskipValue, where lastAG is the allocation group of the last created file.

For example, agskip=3 means each new file will be allocated three AGs away from the starting AG of the most recently created file.

The agskip mount option disables the rotorstep system tunable parameter.

Signed-off-by Rich Johnston <[email protected]>

---
 fs/xfs/xfs_alloc.c      |   35 ++++++++++++++++++++++++++++-------
 fs/xfs/xfs_filestream.c |    8 +++++++-
 fs/xfs/xfs_mount.c      |    1 +
 fs/xfs/xfs_mount.h      |    4 +++-
 fs/xfs/xfs_super.c      |   13 ++++++++++++-
 5 files changed, 51 insertions(+), 10 deletions(-)

Index: b/fs/xfs/xfs_alloc.c
===================================================================

fs/xfs/xfs_alloc.c +0 -4

@@ -2365,12 +2365,20 @@ xfs_alloc_vextent(
 		 * Try near allocation first, then anywhere-in-ag after
 		 * the first a.g. fails.
 		 */
-		if ((args->userdata  == XFS_ALLOC_INITIAL_USER_DATA) &&
-		    (mp->m_flags & XFS_MOUNT_32BITINODES)) {
-			args->fsbno = XFS_AGB_TO_FSB(mp,
-					((mp->m_agfrotor / rotorstep) %

fs/xfs/xfs_alloc.c +0 -1

@@ -2385,8 +2393,21 @@ xfs_alloc_vextent(
 			/*
 			 * Start with the last place we left off.
 			 */
-			args->agno = sagno = (mp->m_agfrotor / rotorstep) %

fs/xfs/xfs_alloc.c +0 -1

@@ -622,7 +622,13 @@ xfs_filestream_associate(
 	 * Set the starting AG using the rotor for inode32, otherwise
 	 * use the directory inode's AG.
 	 */
-	if (mp->m_flags & XFS_MOUNT_32BITINODES) {

Quoted 6 lines · On Tue, Jan 29, 2013 at 09:39:15AM -0600, [email protected] wrote:

The agskip mount option specifies the allocation group, (AG) for a new file, relative to the start of the last created file. agskip has the
opposite effect of the rotorstep system tunable parameter. Each
new file to be placed in the location lastAG + agskipValue, where lastAG is the allocation group of the last created file.

Quoted 2 lines

For example, agskip=3 means each new file will be allocated three AGs away from the starting AG of the most recently created file.

Overall, I'm wondering if this is the right way to approach this problem.

It only really works correctly (in terms of distribution of files/metadata) for fixed volume sizes (i.e. homogenous layouts) - the common case where a skip is useful is after growing a filesystem onto a new volume. It's rare that the new volume is the same as the existing volumes, so it's hard to set a skip value that reliably alternates between old and new volumes.

We talked about this allocation distribution problem last march when we met at LSF, and I thought we agreed that pushing agskip/agrotorstep mount options upstream was not the way we were going to solve this problem after I outlined how I planned to solve this problem.

Indeed, I already have prototype patches for the AG control ioctls that allow the xfs_spaceman program that associate an AG with an "allocation zone" (an "AZ") and partially written kernel patches that implement AZs, per-AZ agi/agf rotors and higher level AZ iteration in the allocator.

(It also allows specification of per-AZ stripe unit/stripe width and other allocation control parameters but those details are little outside the scope of this particular discussion.)

IOWs, I'm close to having a significantly more capable and flexible solution to this problem that doesn't require new mount options to be added or on-disk format changes to be made. As such, I'm not sure that we want to introduce a mount option that is only a partial solution to the problem. We basically can't remove mount options once they have been added, so introducing a new mount option just to mark it deprecated in 6 months time seems like a bad idea....

Hey Dave,

Quoted 7 lines · Dave Chinner

Quoted 6 lines · On Tue, Jan 29, 2013 at 09:39:15AM -0600, [email protected] wrote:

The agskip mount option specifies the allocation group, (AG) for a new file, relative to the start of the last created file. agskip has the
opposite effect of the rotorstep system tunable parameter. Each
new file to be placed in the location lastAG + agskipValue, where lastAG is the allocation group of the last created file.

Quoted 5 lines

Quoted 2 lines

For example, agskip=3 means each new file will be allocated three AGs away from the starting AG of the most recently created file.

Overall, I'm wondering if this is the right way to approach this problem.

We'll have to make sure we all understand the problem we're trying to solve with this before going too far.

Quoted 6 lines

It only really works correctly (in terms of distribution of files/metadata) for fixed volume sizes (i.e. homogenous layouts) - the common case where a skip is useful is after growing a filesystem onto a new volume. It's rare that the new volume is the same as the existing volumes, so it's hard to set a skip value that reliably alternates between old and new volumes.

Based upon what I've read so far on the internal bug when this was introduced, this is more about being able to utilize all allocation groups in a filesystem with many concats.

It's not so much related to balance after growing a
filesystem (which is another interesting problem). The info should be added to this series and be reposted.

Quoted 5 lines

We talked about this allocation distribution problem last march when we met at LSF, and I thought we agreed that pushing agskip/agrotorstep mount options upstream was not the way we were going to solve this problem after I outlined how I planned to solve this problem.

If we can come up with something better, that's great. But AFAICT the problem still needs to be addressed. This is just one way to do it.

Quoted 10 lines · Ben Myers

Hey Dave,

Quoted 7 lines · Dave Chinner

Quoted 6 lines · On Tue, Jan 29, 2013 at 09:39:15AM -0600, [email protected] wrote:

The agskip mount option specifies the allocation group, (AG) for a new file, relative to the start of the last created file. agskip has the
opposite effect of the rotorstep system tunable parameter. Each
new file to be placed in the location lastAG + agskipValue, where lastAG is the allocation group of the last created file.

Quoted 8 lines

Quoted 5 lines

Quoted 2 lines

For example, agskip=3 means each new file will be allocated three AGs away from the starting AG of the most recently created file.

Overall, I'm wondering if this is the right way to approach this problem.

We'll have to make sure we all understand the problem we're trying to solve with this before going too far.

I'm in no doubt about what it is for - I know the exact history of this patch and exactly what problems it was designed to solve because....

Quoted 10 lines

Quoted 6 lines

It only really works correctly (in terms of distribution of files/metadata) for fixed volume sizes (i.e. homogenous layouts) - the common case where a skip is useful is after growing a filesystem onto a new volume. It's rare that the new volume is the same as the existing volumes, so it's hard to set a skip value that reliably alternates between old and new volumes.

Based upon what I've read so far on the internal bug when this was introduced, this is more about being able to utilize all allocation groups in a filesystem with many concats.

... I was still at SGI when that bug was raised and the ag_skip patch was written for the "Enhanced XFS" module in the NAS product SGI was selling at the time. It was written as a quick stopgap by the NAS product engineers to counter the problems being seen on that product due to the nature of the "concat of stripes" storage configuration that product used.

It was never was proposed as an upstream solution because I NACKed it internally. Indeed, at the time I was already well down the path of fixing the problem in XFS in a much more capable way. i.e. this stuff:

http://oss.sgi.com/archives/xfs/2009-02/msg00250.html

http://oss.sgi.com/archives/xfs/2009-02/msg00253.html

And that was planned to replace the agskip hack in the next NAS product release. Unfortunately, once I left SGI nobody picked up the work I was doing and "Enhanced XFS" turned into a zombie. Indeed, agskip was posted back here in 2009 as part of the same code dump as the above patches when the XFS group in Melbourne was let go:

http://oss.sgi.com/archives/xfs/2009-02/msg00252.html

There's a bit of history to this patch ;)

Quoted 3 lines

It's not so much related to balance after growing a
filesystem (which is another interesting problem). The info should be added to this series and be reposted.

Actually, that was one of the problems the patch solved on the NAS product. It was a secondary problem as growing wasn't a common operation, but it was definitely a concern....

Quoted 8 lines

Quoted 5 lines

We talked about this allocation distribution problem last march when we met at LSF, and I thought we agreed that pushing agskip/agrotorstep mount options upstream was not the way we were going to solve this problem after I outlined how I planned to solve this problem.

If we can come up with something better, that's great. But AFAICT the problem still needs to be addressed. This is just one way to do it.

I'm not saynig that it doesn't need to be addressed. I'm just saying the sam ething I said 5 years ago: there's no point pushing it into mainline when far more comprehensive solution is just around the corner....

Cheers,

Dave.

Quoted 11 lines · Dave Chinner

Quoted 10 lines · Ben Myers

Hey Dave,

Quoted 7 lines · Dave Chinner

Quoted 6 lines · On Tue, Jan 29, 2013 at 09:39:15AM -0600, [email protected] wrote:

The agskip mount option specifies the allocation group, (AG) for a new file, relative to the start of the last created file. agskip has the
opposite effect of the rotorstep system tunable parameter. Each
new file to be placed in the location lastAG + agskipValue, where lastAG is the allocation group of the last created file.

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 2 lines

For example, agskip=3 means each new file will be allocated three AGs away from the starting AG of the most recently created file.

Overall, I'm wondering if this is the right way to approach this problem.

I did not see any references to the patchset you referenced below when I was working on submitting this patchset. Thanks for pointing it out.

Quoted 1 line

http://oss.sgi.com/archives/xfs/2009-02/msg00250.html

The patchset (pluggable allocation policies) above looks very promising and I would like to port it to top of tree and use it instead of my agskip proposal. Are there any changes to this patchset we should discuss before I start.

This series will:

1. Add agskip=value mount option to xfs
2. Add agkip mount option test to xfstests
3. Add agskip mount option to the mount.8 manpage (util-linux)

--Rich

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs