Skip to contentSkip to conversation

btrfs: add an autodefrag property

RFC PATCH In review Josef Bacik 4 participants 6 messages Jul 17, 2020 – Jul 19, 2020 Btrfs file system

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 btrfs: add an autodefrag property

Autodefrag is very useful for somethings, like the 9000 sqllite files that Firefox uses, but is way less useful for virt images. Unfortunately this is only available currently as a whole mount option. Fix this by adding an "autodefrag" property, that users can set on a per file or per directory basis. Thus allowing them to control where exactly the extra write activity is going to occur.

---
- This is an RFC because I want to make sure we're ok with this before I go and add btrfs-progs support for this. I'm not married to the name or the value, but I think the core goal is valuable.

 fs/btrfs/btrfs_inode.h |  1 +
 fs/btrfs/file.c        | 16 ++++++++++------
 fs/btrfs/props.c       | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 6 deletions(-)
fs/btrfs/btrfs_inode.h +1 -0
index e7d709505cb1..4f04f0535f90 100644
@@ -31,6 +31,7 @@ enum {
 	BTRFS_INODE_READDIO_NEED_LOCK,
 	BTRFS_INODE_HAS_PROPS,
 	BTRFS_INODE_SNAPSHOT_FLUSH,
+	BTRFS_INODE_AUTODEFRAG,
 };
 
 /* in memory btrfs inode */
fs/btrfs/file.c +10 -6
index 841c516079a9..cac2092bdcdf 100644
@@ -116,12 +116,16 @@ static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
 	return 0;
 }
 
-static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
+static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info,
+				     struct btrfs_inode *inode)
 {
-	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
+	if (btrfs_fs_closing(fs_info))
 		return 0;
 
-	if (btrfs_fs_closing(fs_info))
+	if (inode && test_bit(BTRFS_INODE_AUTODEFRAG, &inode->runtime_flags))
+		return 1;
+
+	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
 		return 0;
 
 	return 1;
@@ -140,7 +144,7 @@ int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
 	u64 transid;
 	int ret;
 
-	if (!__need_auto_defrag(fs_info))
+	if (!__need_auto_defrag(fs_info, inode))
 		return 0;
 
 	if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
@@ -187,7 +191,7 @@ static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 	int ret;
 
-	if (!__need_auto_defrag(fs_info))
+	if (!__need_auto_defrag(fs_info, inode))
 		goto out;
 
 	/*
@@ -348,7 +352,7 @@ int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
 			     &fs_info->fs_state))
 			break;
 
-		if (!__need_auto_defrag(fs_info))
+		if (btrfs_fs_closing(fs_info))
 			break;
 
 		/* find an inode to defrag */
fs/btrfs/props.c +41 -0

53 lines not on this page · raw message

Signature · 1 line
2.24.1
#1 lore raw
Quoted 7 lines · Josef Bacik

Autodefrag is very useful for somethings, like the 9000 sqllite files that Firefox uses, but is way less useful for virt images. Unfortunately this is only available currently as a whole mount option. Fix this by adding an "autodefrag" property, that users can set on a per file or per directory basis. Thus allowing them to control where exactly the extra write activity is going to occur.

Quoted 5 lines

---
- This is an RFC because I want to make sure we're ok with this before I go and add btrfs-progs support for this. I'm not married to the name or the value, but I think the core goal is valuable.

The idea looks pretty good to me.

Although it would be much more convincing to bring some real-world micro bench to show the benefit.

However I still have a concern related to defrag.
Since it's on-disk flag, thus can be inherited by snapshot, then what would happen if an auto-defrag inode get snapshotted.

Would any write to the auto-defrag inode in new snapshot break the space saving?

Quoted 23 lines
 fs/btrfs/btrfs_inode.h |  1 +
 fs/btrfs/file.c        | 16 ++++++++++------
 fs/btrfs/props.c       | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 6 deletions(-)
fs/btrfs/btrfs_inode.h +1 -0
index e7d709505cb1..4f04f0535f90 100644
@@ -31,6 +31,7 @@ enum {
 	BTRFS_INODE_READDIO_NEED_LOCK,
 	BTRFS_INODE_HAS_PROPS,
 	BTRFS_INODE_SNAPSHOT_FLUSH,
+	BTRFS_INODE_AUTODEFRAG,
 };
 
 /* in memory btrfs inode */
fs/btrfs/file.c +0 -0
index 841c516079a9..cac2092bdcdf 100644

fs/btrfs/file.c static int __btrfs_add_inode_defrag(struct btrfs_inode *inode, · line 120

Quoted 6 lines
fs/btrfs/file.c +2 -1
@@ -116,12 +116,16 @@ static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
 	return 0;
 }
 
-static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
+static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info,
+				     struct btrfs_inode *inode)

We can grab fs_info from btrfs_inode, thus no need for the fs_info parameter.

Thanks,
Qu

Quoted 100 lines
fs/btrfs/file.c +8 -5
 {
-	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
+	if (btrfs_fs_closing(fs_info))
 		return 0;
 
-	if (btrfs_fs_closing(fs_info))
+	if (inode && test_bit(BTRFS_INODE_AUTODEFRAG, &inode->runtime_flags))
+		return 1;
+
+	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
 		return 0;
 
 	return 1;
@@ -140,7 +144,7 @@ int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
 	u64 transid;
 	int ret;
 
-	if (!__need_auto_defrag(fs_info))
+	if (!__need_auto_defrag(fs_info, inode))
 		return 0;
 
 	if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
@@ -187,7 +191,7 @@ static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 	int ret;
 
-	if (!__need_auto_defrag(fs_info))
+	if (!__need_auto_defrag(fs_info, inode))
 		goto out;
 
 	/*
@@ -348,7 +352,7 @@ int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
 			     &fs_info->fs_state))
 			break;
 
-		if (!__need_auto_defrag(fs_info))
+		if (btrfs_fs_closing(fs_info))
 			break;
 
 		/* find an inode to defrag */
fs/btrfs/props.c +41 -0
index 2dcb1cb21634..5c6411c8b19f 100644
@@ -310,6 +310,40 @@ static const char *prop_compression_extract(struct inode *inode)
 	return NULL;
 }
 
+static int prop_autodefrag_validate(const char *value, size_t len)
+{
+	if (!value)
+		return 0;
+	if (!strncmp("true", value, 4))
+		return 0;
+	return -EINVAL;
+}
+
+static int prop_autodefrag_apply(struct inode *inode, const char *value,
+				 size_t len)
+{
+	if (len == 0) {
+		clear_bit(BTRFS_INODE_AUTODEFRAG,
+			  &BTRFS_I(inode)->runtime_flags);
+		return 0;
+	}
+
+	if (!strncmp("true", value, 4)) {
+		set_bit(BTRFS_INODE_AUTODEFRAG,
+			&BTRFS_I(inode)->runtime_flags);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static const char *prop_autodefrag_extract(struct inode *inode)
+{
+	if (test_bit(BTRFS_INODE_AUTODEFRAG, &BTRFS_I(inode)->runtime_flags))
+		return "true";
+	return NULL;
+}
+
 static struct prop_handler prop_handlers[] = {
 	{
 		.xattr_name = XATTR_BTRFS_PREFIX "compression",
@@ -318,6 +352,13 @@ static struct prop_handler prop_handlers[] = {
 		.extract = prop_compression_extract,
 		.inheritable = 1
 	},
+	{
+		.xattr_name = XATTR_BTRFS_PREFIX "autodefrag",
+		.validate = prop_autodefrag_validate,
+		.apply = prop_autodefrag_apply,
+		.extract = prop_autodefrag_extract,
+		.inheritable = 1
+	},
 };
 
 static int inherit_props(struct btrfs_trans_handle *trans,
#2 lore raw
Quoted 16 lines · Qu Wenruo
Quoted 7 lines · Josef Bacik

Autodefrag is very useful for somethings, like the 9000 sqllite files that Firefox uses, but is way less useful for virt images. Unfortunately this is only available currently as a whole mount option. Fix this by adding an "autodefrag" property, that users can set on a per file or per directory basis. Thus allowing them to control where exactly the extra write activity is going to occur.

Quoted 5 lines

---
- This is an RFC because I want to make sure we're ok with this before I go and add btrfs-progs support for this. I'm not married to the name or the value, but I think the core goal is valuable.

Quoted 5 lines

The idea looks pretty good to me.

Although it would be much more convincing to bring some real-world micro bench to show the benefit.

The same benefit as what existed originally for autodefrag, firefox isn't unusably slow on spinning rust.

Quoted 7 lines

However I still have a concern related to defrag.
Since it's on-disk flag, thus can be inherited by snapshot, then what would happen if an auto-defrag inode get snapshotted.

Would any write to the auto-defrag inode in new snapshot break the space saving?

Sure but that's the case for all defrag, and this is in fact better than mount -o autodefrag because you can limit the damage. Thanks,

Josef

#3 lore raw
Quoted 29 lines · Josef Bacik
Quoted 18 lines · Qu Wenruo
Quoted 7 lines · Josef Bacik

Autodefrag is very useful for somethings, like the 9000 sqllite files that Firefox uses, but is way less useful for virt images. Unfortunately this is only available currently as a whole mount option. Fix this by adding an "autodefrag" property, that users can set on a per file or per directory basis.  Thus allowing them to control where exactly the extra write activity is going to occur.

Quoted 7 lines

---
- This is an RFC because I want to make sure we're ok with this before I go and
   add btrfs-progs support for this.  I'm not married to the name or the value,
   but I think the core goal is valuable.

Quoted 5 lines

The idea looks pretty good to me.

Although it would be much more convincing to bring some real-world micro bench to show the benefit.

The same benefit as what existed originally for autodefrag, firefox isn't unusably slow on spinning rust.

Quoted 1 line
Quoted 8 lines
Quoted 6 lines

However I still have a concern related to defrag.
Since it's on-disk flag, thus can be inherited by snapshot, then what would happen if an auto-defrag inode get snapshotted.

Would any write to the auto-defrag inode in new snapshot break the space saving?

Sure but that's the case for all defrag, and this is in fact better than mount -o autodefrag because you can limit the damage.  Thanks,

That's right. So no problem at all then.

Thanks,
Qu

Quoted 2 lines

Josef

#4 lore raw
In reply to Josef Bacik
Quoted 39 lines · Josef Bacik

Autodefrag is very useful for somethings, like the 9000 sqllite files that Firefox uses, but is way less useful for virt images. Unfortunately this is only available currently as a whole mount option. Fix this by adding an "autodefrag" property, that users can set on a per file or per directory basis. Thus allowing them to control where exactly the extra write activity is going to occur.

---
- This is an RFC because I want to make sure we're ok with this before I go and add btrfs-progs support for this. I'm not married to the name or the value, but I think the core goal is valuable.

 fs/btrfs/btrfs_inode.h |  1 +
 fs/btrfs/file.c        | 16 ++++++++++------
 fs/btrfs/props.c       | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 6 deletions(-)
fs/btrfs/btrfs_inode.h +1 -0
index e7d709505cb1..4f04f0535f90 100644
@@ -31,6 +31,7 @@ enum {
 	BTRFS_INODE_READDIO_NEED_LOCK,
 	BTRFS_INODE_HAS_PROPS,
 	BTRFS_INODE_SNAPSHOT_FLUSH,
+	BTRFS_INODE_AUTODEFRAG,
 };
 
 /* in memory btrfs inode */
fs/btrfs/file.c +0 -1
index 841c516079a9..cac2092bdcdf 100644
@@ -116,12 +116,16 @@ static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
 	return 0;
 }
 
-static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)

fs/btrfs/file.c static int __btrfs_add_inode_defrag(struct btrfs_inode *inode, · line 126

Quoted 10 lines
fs/btrfs/file.c +5 -2
+static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info,
+				     struct btrfs_inode *inode)
 {
-	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
+	if (btrfs_fs_closing(fs_info))
 		return 0;
 
-	if (btrfs_fs_closing(fs_info))
+	if (inode && test_bit(BTRFS_INODE_AUTODEFRAG, &inode->runtime_flags))
+		return 1;

Qu already made a comment about accessing fs_info from inode, but since you check for the presence of inode here, when do you expect for it to be NULL ? If not, then that check is redundant.

fs/btrfs/file.c static int __btrfs_add_inode_defrag(struct btrfs_inode *inode, · line 131

Quoted 5 lines
fs/btrfs/file.c +2 -0
+
+	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
 		return 0;
 
 	return 1;

nit: while at it use the occasion to convert the function to proper bool

<snip>

#5 lore raw
In reply to Josef Bacik

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA512

Quoted 8 lines · Josef Bacik

Autodefrag is very useful for somethings, like the 9000 sqllite files that Firefox uses, but is way less useful for virt images. Unfortunately this is only available currently as a whole mount option.
Fix this by adding an "autodefrag" property, that users can set on a per
file or per directory basis. Thus allowing them to control where exactly the extra write activity is going to occur.

This would be cool so that openSUSE could drop their autodefrag plugin for zypper (essentially forces defragmentation of rpmdb) and we could write some code to set this property for RPMDB inside RPM itself.

Quoted 141 lines

---
- This is an RFC because I want to make sure we're ok with this before I go and
  add btrfs-progs support for this. I'm not married to the name or the value,
  but I think the core goal is valuable.

 fs/btrfs/btrfs_inode.h |  1 +
 fs/btrfs/file.c        | 16 ++++++++++------
 fs/btrfs/props.c       | 41
+++++++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 6 deletions(-)
fs/btrfs/btrfs_inode.h +1 -0
index e7d709505cb1..4f04f0535f90 100644
@@ -31,6 +31,7 @@ enum {
  	BTRFS_INODE_READDIO_NEED_LOCK,
  	BTRFS_INODE_HAS_PROPS,
  	BTRFS_INODE_SNAPSHOT_FLUSH,
+	BTRFS_INODE_AUTODEFRAG,
  };


 /* in memory btrfs inode */

fs/btrfs/file.c +0 -0
index 841c516079a9..cac2092bdcdf 100644
@@ -116,12 +116,16 @@ static int __btrfs_add_inode_defrag(struct
 btrfs_inode *inode,
  	return 0;
  }

fs/btrfs/file.c +2 -1
-static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
+static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info,
+				     struct btrfs_inode *inode)

 {

fs/btrfs/file.c +1 -1
-	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
+	if (btrfs_fs_closing(fs_info))

  return 0;

fs/btrfs/file.c +1 -1
-	if (btrfs_fs_closing(fs_info))
+	if (inode && test_bit(BTRFS_INODE_AUTODEFRAG, &inode-
Quoted 1 line

runtime_flags))

fs/btrfs/file.c +3 -0
+		return 1;
+
+	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))

  return 0;

  return 1;

fs/btrfs/file.c +0 -0
@@ -140,7 +144,7 @@ int btrfs_add_inode_defrag(struct
 btrfs_trans_handle *trans,
  	u64 transid;
  	int ret;

fs/btrfs/file.c +1 -1
-	if (!__need_auto_defrag(fs_info))
+	if (!__need_auto_defrag(fs_info, inode))

  return 0;

  if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))

fs/btrfs/file.c +0 -0
@@ -187,7 +191,7 @@ static void btrfs_requeue_inode_defrag(struct
 btrfs_inode *inode,
  	struct btrfs_fs_info *fs_info = inode->root->fs_info;
  	int ret;

fs/btrfs/file.c +1 -1
-	if (!__need_auto_defrag(fs_info))
+	if (!__need_auto_defrag(fs_info, inode))

  goto out;

  /*

fs/btrfs/file.c +0 -0
@@ -348,7 +352,7 @@ int btrfs_run_defrag_inodes(struct btrfs_fs_info
 *fs_info)
  			     &fs_info->fs_state))
  			break;

fs/btrfs/file.c +1 -1
-		if (!__need_auto_defrag(fs_info))
+		if (btrfs_fs_closing(fs_info))

  break;

  /* find an inode to defrag */

fs/btrfs/props.c +0 -0
index 2dcb1cb21634..5c6411c8b19f 100644
@@ -310,6 +310,40 @@ static const char
 *prop_compression_extract(struct inode *inode)
  	return NULL;
  }

fs/btrfs/props.c +10 -0
+static int prop_autodefrag_validate(const char *value, size_t len)
+{
+	if (!value)
+		return 0;
+	if (!strncmp("true", value, 4))
+		return 0;
+	return -EINVAL;
+}
+
+static int prop_autodefrag_apply(struct inode *inode, const char

*value,

fs/btrfs/props.c +20 -0
+				 size_t len)
+{
+	if (len == 0) {
+		clear_bit(BTRFS_INODE_AUTODEFRAG,
+			  &BTRFS_I(inode)->runtime_flags);
+		return 0;
+	}
+
+	if (!strncmp("true", value, 4)) {
+		set_bit(BTRFS_INODE_AUTODEFRAG,
+			&BTRFS_I(inode)->runtime_flags);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static const char *prop_autodefrag_extract(struct inode *inode)
+{
+	if (test_bit(BTRFS_INODE_AUTODEFRAG, &BTRFS_I(inode)-
Quoted 1 line

runtime_flags))

fs/btrfs/props.c +4 -0
+		return "true";
+	return NULL;
+}
+

 static struct prop_handler prop_handlers[] = {
  {
  .xattr_name = XATTR_BTRFS_PREFIX "compression",

fs/btrfs/props.c +0 -0
@@ -318,6 +352,13 @@ static struct prop_handler prop_handlers[] = {
  		.extract = prop_compression_extract,

  .inheritable = 1
  },

fs/btrfs/props.c +7 -0
+	{
+		.xattr_name = XATTR_BTRFS_PREFIX "autodefrag",
+		.validate = prop_autodefrag_validate,
+		.apply = prop_autodefrag_apply,
+		.extract = prop_autodefrag_extract,
+		.inheritable = 1
+	},
Quoted 3 lines

 };

 static int inherit_props(struct btrfs_trans_handle *trans,

- --
Igor Raits <[email protected]>
-----BEGIN PGP SIGNATURE-----

iQIzBAEBCgAdFiEEcwgJ58gsbV5f5dMcEV1auJxcHh4FAl8UXqgACgkQEV1auJxc Hh7QpxAAmC5pEaP1WTQQgfb+tS+pww2ln+Vk/3yTBRjOapyvisk7FIs4evPLlKFW kIGb2q+FsGTZ+f/8PWXEcXJSU2HSXYv3dbHh6gp6/ZFKMjZPAoxnErFTDx2ZT1af 0GcmKJdpV8RS24qQjs+cORnzOPF1HAyBSL6qHEfpdzkws1LmJXvPOxsToVV24h1D JKaCH3XdIRLf4JXuLyRup0ASpspA1oENHLr6+YcoQjN8IVTNjPaBdJcz+VkFarEi tIwWMJr3A6e1yFWkKldt3HBMtMR+QLWYga3b8d2tROZ2bAsU+JXeWtyUmBONx++r japV9wSquPEj7ht2QH58VsisU68tSFHqyyaGJ5a6CV6BsMkF5KhArK971F3Du3xl te8A5HVvSOnNLzHllhklMehDN2e0qsTjeZaU9DniCV1OkS5F8SNXftJ25YWJtrba mdchFezuXwcQzfC/G5WSZK+6YDNb/Gnind+rkV8fgtz7JiuVpluR3EMZMVD0mMN8 ahZH86xw+78VZeQ/4Q1FGjS+BrDVakn8cGKjSUjJAFF+owx1aXIPByObIKAKd6rf lEmsaPWj0roM81PU+93BxN16Ti8s9wzEM1eoBEmhVrFCKsKt7iSJ+ZJyLRpPIRCZ qAexm0iOKCxY8EMa0A7XbCxcvDSw6NXjwgbkNqFQRd4286uDdsk= =kcj9
-----END PGP SIGNATURE-----