Skip to contentSkip to conversation

ceph: support cross-quota-tree renames

PATCH Superseded Luis Henriques 2 participants 5 messages Apr 6, 2020 – Apr 7, 2020 Ceph client

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover ceph: support cross-quota-tree renames

Hi!

The following patches will make the cephfs kernel client behave the same way as the fuse client when doing renames across different quota realms.

Luis Henriques (2):
ceph: normalize 'delta' parameter usage in check_quota_exceeded
ceph: allow rename operation under different quota realms

 fs/ceph/dir.c   |  9 +++----
 fs/ceph/quota.c | 62 +++++++++++++++++++++++++++++++++++++++++++++----
 fs/ceph/super.h |  3 ++-
 3 files changed, 65 insertions(+), 9 deletions(-)
#1 lore raw

PATCH 1/2 ceph: normalize 'delta' parameter usage in check_quota_exceeded

Function check_quota_exceeded() uses delta parameter only for the QUOTA_CHECK_MAX_BYTES_OP operation. Using this parameter also for MAX_FILES will makes the code cleaner and will be required to support cross-quota-tree renames.

---
 fs/ceph/quota.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
fs/ceph/quota.c +1 -3
index de56dee60540..c5c8050f0f99 100644
@@ -361,8 +361,6 @@ static bool check_quota_exceeded(struct inode *inode, enum quota_check_op op,
 		spin_unlock(&ci->i_ceph_lock);
 		switch (op) {
 		case QUOTA_CHECK_MAX_FILES_OP:
-			exceeded = (max && (rvalue >= max));
-			break;
 		case QUOTA_CHECK_MAX_BYTES_OP:
 			exceeded = (max && (rvalue + delta > max));
 			break;
@@ -417,7 +415,7 @@ bool ceph_quota_is_max_files_exceeded(struct inode *inode)
 
 	WARN_ON(!S_ISDIR(inode->i_mode));
 
-	return check_quota_exceeded(inode, QUOTA_CHECK_MAX_FILES_OP, 0);
+	return check_quota_exceeded(inode, QUOTA_CHECK_MAX_FILES_OP, 1);
 }
 
 /*
#2 lore raw

PATCH 2/2 ceph: allow rename operation under different quota realms

In reply to Luis Henriques

Returning -EXDEV when trying to 'mv' files/directories from different quota realms results in copy+unlink operations instead of the faster CEPH_MDS_OP_RENAME. This will occur even when there aren't any quotas set in the destination directory, or if there's enough space left for the new file(s).

This patch adds a new helper function to be called on rename operations which will allow these operations if they can be executed. This patch mimics userland fuse client commit b8954e5734b3 ("client:
optimize rename operation under different quota root").

Since ceph_quota_is_same_realm() is now called only from this new helper, make it static.

URL: https://tracker.ceph.com/issues/44791

---
 fs/ceph/dir.c   |  9 ++++----
 fs/ceph/quota.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++-
 fs/ceph/super.h |  3 ++-
 3 files changed, 64 insertions(+), 6 deletions(-)
fs/ceph/dir.c +5 -4
index d0cd0aba5843..9d3f0062d800 100644
@@ -1099,11 +1099,12 @@ static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry,
 			op = CEPH_MDS_OP_RENAMESNAP;
 		else
 			return -EROFS;
+	} else {
+		err = ceph_quota_check_rename(mdsc, d_inode(old_dentry),
+					      new_dir);
+		if (err)
+			return err;
 	}
-	/* don't allow cross-quota renames */
-	if ((old_dir != new_dir) &&
-	    (!ceph_quota_is_same_realm(old_dir, new_dir)))
-		return -EXDEV;
 
 	dout("rename dir %p dentry %p to dir %p dentry %p\n",
 	     old_dir, old_dentry, new_dir, new_dentry);
fs/ceph/quota.c +57 -1

67 lines not on this page · raw message

fs/ceph/super.h +2 -1
index 037cdfb2ad4f..d5853831a6b5 100644
@@ -1175,13 +1175,14 @@ extern void ceph_handle_quota(struct ceph_mds_client *mdsc,
 			      struct ceph_mds_session *session,
 			      struct ceph_msg *msg);
 extern bool ceph_quota_is_max_files_exceeded(struct inode *inode);
-extern bool ceph_quota_is_same_realm(struct inode *old, struct inode *new);
 extern bool ceph_quota_is_max_bytes_exceeded(struct inode *inode,
 					     loff_t newlen);
 extern bool ceph_quota_is_max_bytes_approaching(struct inode *inode,
 						loff_t newlen);
 extern bool ceph_quota_update_statfs(struct ceph_fs_client *fsc,
 				     struct kstatfs *buf);
+extern int ceph_quota_check_rename(struct ceph_mds_client *mdsc,
+				   struct inode *old, struct inode *new);
 extern void ceph_cleanup_quotarealms_inodes(struct ceph_mds_client *mdsc);
 
 #endif /* _FS_CEPH_SUPER_H */
#3 lore raw

Re: [PATCH 2/2] ceph: allow rename operation under different quota realms

Quoted 27 lines · Luis Henriques

Returning -EXDEV when trying to 'mv' files/directories from different quota realms results in copy+unlink operations instead of the faster CEPH_MDS_OP_RENAME. This will occur even when there aren't any quotas set in the destination directory, or if there's enough space left for the new file(s).

This patch adds a new helper function to be called on rename operations which will allow these operations if they can be executed. This patch mimics userland fuse client commit b8954e5734b3 ("client:
optimize rename operation under different quota root").

Since ceph_quota_is_same_realm() is now called only from this new helper, make it static.

URL: https://tracker.ceph.com/issues/44791

---
 fs/ceph/dir.c   |  9 ++++----
 fs/ceph/quota.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++-
 fs/ceph/super.h |  3 ++-
 3 files changed, 64 insertions(+), 6 deletions(-)
fs/ceph/dir.c +0 -0
index d0cd0aba5843..9d3f0062d800 100644

fs/ceph/dir.c static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry, · line 1104

Quoted 6 lines
fs/ceph/dir.c +3 -0
@@ -1099,11 +1099,12 @@ static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry,
 			op = CEPH_MDS_OP_RENAMESNAP;
 		else
 			return -EROFS;
+	} else {
+		err = ceph_quota_check_rename(mdsc, d_inode(old_dentry),
+					      new_dir);

I was wondering why not use "old_dir" here, but I think this is more correct. I guess a directory could have a different quotarealm from its parent?

Quoted 40 lines
fs/ceph/dir.c +2 -4
+		if (err)
+			return err;
 	}
-	/* don't allow cross-quota renames */
-	if ((old_dir != new_dir) &&
-	    (!ceph_quota_is_same_realm(old_dir, new_dir)))
-		return -EXDEV;
 
 	dout("rename dir %p dentry %p to dir %p dentry %p\n",
 	     old_dir, old_dentry, new_dir, new_dentry);
fs/ceph/quota.c +14 -1
index c5c8050f0f99..a6dd1a528c70 100644
@@ -264,7 +264,7 @@ static struct ceph_snap_realm *get_quota_realm(struct ceph_mds_client *mdsc,
 	return NULL;
 }
 
-bool ceph_quota_is_same_realm(struct inode *old, struct inode *new)
+static bool ceph_quota_is_same_realm(struct inode *old, struct inode *new)
 {
 	struct ceph_mds_client *mdsc = ceph_inode_to_client(old)->mdsc;
 	struct ceph_snap_realm *old_realm, *new_realm;
@@ -516,3 +516,59 @@ bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf)
 	return is_updated;
 }
 
+/*
+ * ceph_quota_check_rename - check if a rename can be executed
+ * @mdsc:	MDS client instance
+ * @old:	inode to be copied
+ * @new:	destination inode (directory)
+ *
+ * This function verifies if a rename (e.g. moving a file or directory) can be
+ * executed.  It forces an rstat update in the @new target directory (and in the
+ * source @old as well, if it's a directory).  The actual check is done both for
+ * max_files and max_bytes.
+ *
+ * This function returns 0 if it's OK to do the rename, or, if quotas are
+ * exceeded, -EXDEV (if @old is a directory) or -EDQUOT.

fs/ceph/quota.c bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf) · line 541

Quoted 10 lines
fs/ceph/quota.c +10 -0
+ */
+int ceph_quota_check_rename(struct ceph_mds_client *mdsc,
+			    struct inode *old, struct inode *new)
+{
+	struct ceph_inode_info *ci_old = ceph_inode(old);
+	int ret = 0;
+
+	if ((old == new) || (ceph_quota_is_same_realm(old, new)))
+		return 0;
+

"old" represents the old dentry being moved. "new" is the new parent dir. Do we need to test for old == new? The vfs won't allow the source to be the ancestor of the target (or vice versa). From vfs_rename():

        /* source should not be ancestor of target */
        error = -EINVAL;
        if (old_dentry == trap)
                goto exit5;
        /* target should not be an ancestor of source */
        if (!(flags & RENAME_EXCHANGE))
                error = -ENOTEMPTY;
Quoted 43 lines
fs/ceph/quota.c +33 -0
+	/*
+	 * Get the latest rstat for target directory (and for source, if a
+	 * directory)
+	 */
+	ret = ceph_do_getattr(new, CEPH_STAT_RSTAT, false);
+	if (ret)
+		return ret;
+
+	if (S_ISDIR(old->i_mode)) {
+		ret = ceph_do_getattr(old, CEPH_STAT_RSTAT, false);
+		if (ret)
+			return ret;
+		ret = check_quota_exceeded(new, QUOTA_CHECK_MAX_BYTES_OP,
+					   ci_old->i_rbytes);
+		if (!ret)
+			ret = check_quota_exceeded(new,
+						   QUOTA_CHECK_MAX_FILES_OP,
+						   ci_old->i_rfiles +
+						   ci_old->i_rsubdirs);
+		if (ret)
+			ret = -EXDEV;
+	} else {
+		ret = check_quota_exceeded(new, QUOTA_CHECK_MAX_BYTES_OP,
+					   i_size_read(old));
+		if (!ret)
+			ret = check_quota_exceeded(new,
+						   QUOTA_CHECK_MAX_FILES_OP, 1);
+		if (ret)
+			ret = -EDQUOT;
+	}
+
+	return ret;
+}
fs/ceph/super.h +0 -1
index 037cdfb2ad4f..d5853831a6b5 100644
@@ -1175,13 +1175,14 @@ extern void ceph_handle_quota(struct ceph_mds_client *mdsc,
 			      struct ceph_mds_session *session,
 			      struct ceph_msg *msg);
 extern bool ceph_quota_is_max_files_exceeded(struct inode *inode);
-extern bool ceph_quota_is_same_realm(struct inode *old, struct inode *new);
 extern bool ceph_quota_is_max_bytes_exceeded(struct inode *inode,

fs/ceph/super.h extern void ceph_handle_quota(struct ceph_mds_client *mdsc, · line 1188

Quoted 10 lines
fs/ceph/super.h +2 -0
 					     loff_t newlen);
 extern bool ceph_quota_is_max_bytes_approaching(struct inode *inode,
 						loff_t newlen);
 extern bool ceph_quota_update_statfs(struct ceph_fs_client *fsc,
 				     struct kstatfs *buf);
+extern int ceph_quota_check_rename(struct ceph_mds_client *mdsc,
+				   struct inode *old, struct inode *new);
 extern void ceph_cleanup_quotarealms_inodes(struct ceph_mds_client *mdsc);
 
 #endif /* _FS_CEPH_SUPER_H */

Looks good otherwise. Nice work!

Signature · 1 line
Jeff Layton <[email protected]>
#4 lore raw

Re: [PATCH 2/2] ceph: allow rename operation under different quota realms

Quoted 23 lines · Jeff Layton
Quoted 22 lines · Luis Henriques

Returning -EXDEV when trying to 'mv' files/directories from different quota realms results in copy+unlink operations instead of the faster CEPH_MDS_OP_RENAME. This will occur even when there aren't any quotas set in the destination directory, or if there's enough space left for the new file(s).

This patch adds a new helper function to be called on rename operations which will allow these operations if they can be executed. This patch mimics userland fuse client commit b8954e5734b3 ("client:
optimize rename operation under different quota root").

Since ceph_quota_is_same_realm() is now called only from this new helper, make it static.

URL: https://tracker.ceph.com/issues/44791

---
 fs/ceph/dir.c   |  9 ++++----
 fs/ceph/quota.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++-
 fs/ceph/super.h |  3 ++-
 3 files changed, 64 insertions(+), 6 deletions(-)
Quoted 15 lines

fs/ceph/dir.c static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry, · line 1104

Quoted 11 lines
fs/ceph/dir.c +3 -0
index d0cd0aba5843..9d3f0062d800 100644
@@ -1099,11 +1099,12 @@ static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry,
 			op = CEPH_MDS_OP_RENAMESNAP;
 		else
 			return -EROFS;
+	} else {
+		err = ceph_quota_check_rename(mdsc, d_inode(old_dentry),
+					      new_dir);

I was wondering why not use "old_dir" here, but I think this is more correct. I guess a directory could have a different quotarealm from its parent?

Yes, you can set, for example, ceph.quota.max_files to in a directory, and set it again in a subdirectory; this subdirectory will now be on a different quotarealm.

Quoted 56 lines
Quoted 41 lines
fs/ceph/dir.c +2 -4
+		if (err)
+			return err;
 	}
-	/* don't allow cross-quota renames */
-	if ((old_dir != new_dir) &&
-	    (!ceph_quota_is_same_realm(old_dir, new_dir)))
-		return -EXDEV;
 
 	dout("rename dir %p dentry %p to dir %p dentry %p\n",
 	     old_dir, old_dentry, new_dir, new_dentry);
fs/ceph/quota.c +14 -1
index c5c8050f0f99..a6dd1a528c70 100644
@@ -264,7 +264,7 @@ static struct ceph_snap_realm *get_quota_realm(struct ceph_mds_client *mdsc,
 	return NULL;
 }
 
-bool ceph_quota_is_same_realm(struct inode *old, struct inode *new)
+static bool ceph_quota_is_same_realm(struct inode *old, struct inode *new)
 {
 	struct ceph_mds_client *mdsc = ceph_inode_to_client(old)->mdsc;
 	struct ceph_snap_realm *old_realm, *new_realm;
@@ -516,3 +516,59 @@ bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf)
 	return is_updated;
 }
 
+/*
+ * ceph_quota_check_rename - check if a rename can be executed
+ * @mdsc:	MDS client instance
+ * @old:	inode to be copied
+ * @new:	destination inode (directory)
+ *
+ * This function verifies if a rename (e.g. moving a file or directory) can be
+ * executed.  It forces an rstat update in the @new target directory (and in the
+ * source @old as well, if it's a directory).  The actual check is done both for
+ * max_files and max_bytes.
+ *
+ * This function returns 0 if it's OK to do the rename, or, if quotas are
+ * exceeded, -EXDEV (if @old is a directory) or -EDQUOT.

fs/ceph/quota.c bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf) · line 541

Quoted 10 lines
fs/ceph/quota.c +10 -0
+ */
+int ceph_quota_check_rename(struct ceph_mds_client *mdsc,
+			    struct inode *old, struct inode *new)
+{
+	struct ceph_inode_info *ci_old = ceph_inode(old);
+	int ret = 0;
+
+	if ((old == new) || (ceph_quota_is_same_realm(old, new)))
+		return 0;
+

"old" represents the old dentry being moved. "new" is the new parent dir. Do we need to test for old == new? The vfs won't allow the source to be the ancestor of the target (or vice versa). From vfs_rename():

Quoted 8 lines
        /* source should not be ancestor of target */
        error = -EINVAL;
        if (old_dentry == trap)
                goto exit5;
        /* target should not be an ancestor of source */
        if (!(flags & RENAME_EXCHANGE))
                error = -ENOTEMPTY;

Err... yeah, that 'old == new' is a mistake. I just wanted to keep the optimization in the original code:

	if ((old_dir != new_dir) &&
	    (!ceph_quota_is_same_realm(old_dir, new_dir)))
		return -EXDEV;

I'll send out v2 in a minute with the following change to ceph_rename:

	} else if (old_dir != new_dir) {
		err = ceph_quota_check_rename(mdsc, d_inode(old_dentry),
					      new_dir);

Cheers,
--
Luís

Quoted 58 lines
Quoted 43 lines
fs/ceph/quota.c +33 -0
+	/*
+	 * Get the latest rstat for target directory (and for source, if a
+	 * directory)
+	 */
+	ret = ceph_do_getattr(new, CEPH_STAT_RSTAT, false);
+	if (ret)
+		return ret;
+
+	if (S_ISDIR(old->i_mode)) {
+		ret = ceph_do_getattr(old, CEPH_STAT_RSTAT, false);
+		if (ret)
+			return ret;
+		ret = check_quota_exceeded(new, QUOTA_CHECK_MAX_BYTES_OP,
+					   ci_old->i_rbytes);
+		if (!ret)
+			ret = check_quota_exceeded(new,
+						   QUOTA_CHECK_MAX_FILES_OP,
+						   ci_old->i_rfiles +
+						   ci_old->i_rsubdirs);
+		if (ret)
+			ret = -EXDEV;
+	} else {
+		ret = check_quota_exceeded(new, QUOTA_CHECK_MAX_BYTES_OP,
+					   i_size_read(old));
+		if (!ret)
+			ret = check_quota_exceeded(new,
+						   QUOTA_CHECK_MAX_FILES_OP, 1);
+		if (ret)
+			ret = -EDQUOT;
+	}
+
+	return ret;
+}
fs/ceph/super.h +0 -1
index 037cdfb2ad4f..d5853831a6b5 100644
@@ -1175,13 +1175,14 @@ extern void ceph_handle_quota(struct ceph_mds_client *mdsc,
 			      struct ceph_mds_session *session,
 			      struct ceph_msg *msg);
 extern bool ceph_quota_is_max_files_exceeded(struct inode *inode);
-extern bool ceph_quota_is_same_realm(struct inode *old, struct inode *new);
 extern bool ceph_quota_is_max_bytes_exceeded(struct inode *inode,

fs/ceph/super.h extern void ceph_handle_quota(struct ceph_mds_client *mdsc, · line 1188

Quoted 10 lines
fs/ceph/super.h +2 -0
 					     loff_t newlen);
 extern bool ceph_quota_is_max_bytes_approaching(struct inode *inode,
 						loff_t newlen);
 extern bool ceph_quota_update_statfs(struct ceph_fs_client *fsc,
 				     struct kstatfs *buf);
+extern int ceph_quota_check_rename(struct ceph_mds_client *mdsc,
+				   struct inode *old, struct inode *new);
 extern void ceph_cleanup_quotarealms_inodes(struct ceph_mds_client *mdsc);
 
 #endif /* _FS_CEPH_SUPER_H */

Looks good otherwise. Nice work!

Signature · 1 line
Jeff Layton <[email protected]>