Skip to contentSkip to conversation

fix quota subdir mounts

PATCH New Luis Henriques 1 participant 3 messages Mar 21, 2019 Ceph client

lore
Share & tools
lore
#0 lore raw

v3 cover fix quota subdir mounts

Hi,

As recently reported in the ceph-users mailing-list[1], the kernel client behaves differently from the fuse client regarding mounting subdirs where quotas are in effect. I've also created a bug to track this issue[2].

The following patches are a possible way of fixing this issue. The performance impact should be close to zero if the mount is done in the CephFS root inode. When we're mounting subdirs, we may have extra queries to the MDSs, depending on how many extra realms we'll need to loop through.

Changes since v2:

- Replaced inodes list in mdsc by an rbtree. This is because we need to able to keep track errors in lookupino so that we don't keep sending the same useless request for inodes that have failed. This also resulted in reworking the locking in lookup_quotarealm_inode, so that 2 threads can lookup the same inode at the same time

- No need to set realm->inode in lookup_quotarealm_inode() as the lookupino has set it already.

Changes since v1:

- Loop to free mdsc->quotarealms_inodes_list list was moved further down where it's not possible to race with insertions. This way there's no need to hold the spinlock anymore.

- Clarified comments regarding the get_quota_realm function 'retry' parameter, both in the function itself and in function ceph_quota_is_same_realm, where that param is set to 'false'

- Distinguish between 'realm->inode is NULL' and igrab failures, both in get_quota_realm and check_quota_exceeded

Changes since RFC:

The 1st patch hasn't been changed since the initial RFC. The 2nd patch has been refactored to include the following changes:

- Zheng Yan's suggestions, i.e, move inode references from the realms to ceph_mds_client instance

- It now also handles other cases where an MDS lookup may need to be performed:
* statfs when there are quotas
* renames, to forbid cross-quota renames

[1] http://lists.ceph.com/pipermail/ceph-users-ceph.com/2019-February/033357.html
[2] https://tracker.ceph.com/issues/38482

Cheers,
--
Luís

Luis Henriques (2):
ceph: factor out ceph_lookup_inode()
ceph: quota: fix quota subdir mounts

 fs/ceph/export.c     |  14 +++-
 fs/ceph/mds_client.c |  19 ++++++
 fs/ceph/mds_client.h |  18 +++++
 fs/ceph/quota.c      | 158 ++++++++++++++++++++++++++++++++++++++++---
 fs/ceph/super.h      |   1 +
 5 files changed, 199 insertions(+), 11 deletions(-)
#1 lore raw

PATCH v3 1/2 ceph: factor out ceph_lookup_inode()

This function will be used by __fh_to_dentry and by the quotas code, to find quota realm inodes that are not visible in the mountpoint.

---
 fs/ceph/export.c | 14 +++++++++++++-
 fs/ceph/super.h  |  1 +
 2 files changed, 14 insertions(+), 1 deletion(-)
fs/ceph/export.c +13 -1
index 3c59ad180ef0..0d8ead82c816 100644
@@ -59,7 +59,7 @@ static int ceph_encode_fh(struct inode *inode, u32 *rawfh, int *max_len,
 	return type;
 }
 
-static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino)
+struct inode *ceph_lookup_inode(struct super_block *sb, u64 ino)
 {
 	struct ceph_mds_client *mdsc = ceph_sb_to_client(sb)->mdsc;
 	struct inode *inode;
@@ -92,12 +92,24 @@ static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino)
 		ceph_mdsc_put_request(req);
 		if (!inode)
 			return ERR_PTR(-ESTALE);
+		if (err)
+			return ERR_PTR(err);
 		if (inode->i_nlink == 0) {
 			iput(inode);
 			return ERR_PTR(-ESTALE);
 		}
 	}
 
+	return inode;
+}
+
+static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino)
+{
+	struct inode *inode = ceph_lookup_inode(sb, ino);
+
+	if (IS_ERR(inode))
+		return ERR_CAST(inode);
+
 	return d_obtain_alias(inode);
 }
 
fs/ceph/super.h +1 -0
index 16c03188578e..976f200164f9 100644
@@ -1082,6 +1082,7 @@ extern long ceph_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 
 /* export.c */
 extern const struct export_operations ceph_export_ops;
+struct inode *ceph_lookup_inode(struct super_block *sb, u64 ino);
 
 /* locks.c */
 extern __init void ceph_flock_init(void);
#2 lore raw

PATCH v3 2/2 ceph: quota: fix quota subdir mounts

In reply to Luis Henriques

The CephFS kernel client does not enforce quotas set in a directory that isn't visible from the mount point. For example, given the path '/dir1/dir2', if quotas are set in 'dir1' and the filesystem is mounted with

mount -t ceph <server>:<port>:/dir1/ /mnt

then the client won't be able to access 'dir1' inode, even if 'dir2' belongs to a quota realm that points to it.

This patch fixes this issue by simply doing an MDS LOOKUPINO operation for unknown inodes. Any inode reference obtained this way will be added to a list in ceph_mds_client, and will only be released when the filesystem is umounted.

---
 fs/ceph/mds_client.c |  19 ++++++
 fs/ceph/mds_client.h |  18 +++++
 fs/ceph/quota.c      | 158 ++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 185 insertions(+), 10 deletions(-)
fs/ceph/mds_client.c +19 -0
index 21c33ed048ed..0d49000f4daa 100644
@@ -4077,6 +4077,8 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)
 	mdsc->max_sessions = 0;
 	mdsc->stopping = 0;
 	atomic64_set(&mdsc->quotarealms_count, 0);
+	mdsc->quotarealms_inodes = RB_ROOT;
+	mutex_init(&mdsc->quotarealms_inodes_mutex);
 	mdsc->last_snap_seq = 0;
 	init_rwsem(&mdsc->snap_rwsem);
 	mdsc->snap_realms = RB_ROOT;
@@ -4156,6 +4158,9 @@ static void wait_requests(struct ceph_mds_client *mdsc)
  */
 void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc)
 {
+	struct ceph_quotarealm_inode *qri;
+	struct rb_node *node;
+
 	dout("pre_umount\n");
 	mdsc->stopping = 1;
 
@@ -4168,6 +4173,20 @@ void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc)
 	 * their inode/dcache refs
 	 */
 	ceph_msgr_flush();
+	/*
+	 * It should now be safe to clean quotarealms_inode tree without holding
+	 * mdsc->quotarealms_inodes_mutex...
+	 */
+	mutex_lock(&mdsc->quotarealms_inodes_mutex);
+	while (!RB_EMPTY_ROOT(&mdsc->quotarealms_inodes)) {
+		node = rb_first(&mdsc->quotarealms_inodes);
+		qri = rb_entry(node, struct ceph_quotarealm_inode, node);
+		rb_erase(node, &mdsc->quotarealms_inodes);
+		if (qri->ci)
+			iput(&qri->ci->vfs_inode);
+		kfree(qri);
+	}
+	mutex_unlock(&mdsc->quotarealms_inodes_mutex);
 }
 
 /*
fs/ceph/mds_client.h +18 -0
index 50385a481fdb..9cfa90b4a6d9 100644
@@ -325,6 +325,18 @@ struct ceph_snapid_map {
 	unsigned long last_used;
 };
 
+/*
+ * node for list of quotarealm inodes that are not visible from the filesystem
+ * mountpoint, but required to handle, e.g. quotas.
+ */
+struct ceph_quotarealm_inode {
+	struct rb_node node;
+	struct ceph_inode_info *ci;
+	struct mutex mutex;
+	u64 ino;
+	unsigned long timeout; /* last time a lookup failed for this inode */
+};
+
 /*
  * mds client state
  */
@@ -344,6 +356,12 @@ struct ceph_mds_client {
 	int                     stopping;      /* true if shutting down */
 
 	atomic64_t		quotarealms_count; /* # realms with quota */
+	/*
+	 * We keep a list of inodes we don't see in the mountpoint but that we
+	 * need to track quota realms.
+	 */
+	struct rb_root		quotarealms_inodes;
+	struct mutex		quotarealms_inodes_mutex;
 
 	/*
 	 * snap_rwsem will cover cap linkage into snaprealms, and
fs/ceph/quota.c +148 -10

221 lines not on this page · raw message