Skip to contentSkip to conversation

fs, ipc: Use an asynchronous version of kern_unmount in IPC

PATCH In review Salman Qazi 3 participants 6 messages Feb 6, 2019 – Mar 4, 2019 VFS

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 fs, ipc: Use an asynchronous version of kern_unmount in IPC

Prior to this patch, the kernel can spend a lot of time with this stack trace:

[<ffffffffbe5491e3>] __wait_rcu_gp+0x93/0xe0
[<ffffffffbe549418>] synchronize_sched+0x48/0x60
[<ffffffffbe7ae5b3>] kern_unmount+0x3a/0x46
[<ffffffffbe847c02>] mq_put_mnt+0x15/0x17
[<ffffffffbe8481af>] put_ipc_ns+0x36/0x8b

This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.

Since mntput() sleeps, it needs to be deferred to a work queue.

Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.

Tested: Ran the following program:

    int main(void)
    {
            int pid;
            int status;
            int i;

            for (i = 0; i < 1000; i++) {
                    pid = fork();
                    if (!pid) {
                            assert(!unshare(CLONE_NEWUSER|
                                      CLONE_NEWIPC|CLONE_NEWNS));
                            return 0;
                    }

                    assert(waitpid(pid, &status, 0) == pid);
            }
    }

Before:

$ time ./unshare2
real    0m9.784s
user    0m0.428s
sys     0m0.000s

After:

$ time ./unshare2
real    0m0.368s
user    0m0.226s
sys     0m0.122s
---
 fs/namespace.c     | 41 +++++++++++++++++++++++++++++++++++++++++
 include/linux/fs.h |  1 +
 ipc/mqueue.c       |  2 +-
 3 files changed, 43 insertions(+), 1 deletion(-)
fs/namespace.c +41 -0
index a677b59efd74..caa51ca81605 100644
@@ -3323,6 +3323,47 @@ void kern_unmount(struct vfsmount *mnt)
 }
 EXPORT_SYMBOL(kern_unmount);
 
+struct async_unmount_cb {
+	struct vfsmount *mnt;
+	struct work_struct work;
+	struct rcu_head rcu_head;
+};
+
+static void kern_unmount_work(struct work_struct *work)
+{
+	struct async_unmount_cb *cb = container_of(work,
+			struct async_unmount_cb, work);
+
+	mntput(cb->mnt);
+	kfree(cb);
+}
+
+static void kern_unmount_rcu_cb(struct rcu_head *rcu_head)
+{
+	struct async_unmount_cb *cb = container_of(rcu_head,
+			struct async_unmount_cb, rcu_head);
+
+	INIT_WORK(&cb->work, kern_unmount_work);
+	schedule_work(&cb->work);
+
+}
+
+void kern_unmount_async(struct vfsmount *mnt)
+{
+	/* release long term mount so mount point can be released */
+	if (!IS_ERR_OR_NULL(mnt)) {
+		struct async_unmount_cb *cb = kmalloc(sizeof(*cb), GFP_KERNEL);
+
+		if (cb) {
+			real_mount(mnt)->mnt_ns = NULL;
+			cb->mnt = mnt;
+			call_rcu(&cb->rcu_head, kern_unmount_rcu_cb);
+		} else {
+			kern_unmount(mnt);
+		}
+	}
+}
+
 bool our_mnt(struct vfsmount *mnt)
 {
 	return check_mnt(real_mount(mnt));
include/linux/fs.h +1 -0
index 29d8e2cfed0e..8865997a8722 100644
@@ -2274,6 +2274,7 @@ extern int register_filesystem(struct file_system_type *);
 extern int unregister_filesystem(struct file_system_type *);
 extern struct vfsmount *kern_mount_data(struct file_system_type *, void *data);
 #define kern_mount(type) kern_mount_data(type, NULL)
+extern void kern_unmount_async(struct vfsmount *mnt);
 extern void kern_unmount(struct vfsmount *mnt);
 extern int may_umount_tree(struct vfsmount *);
 extern int may_umount(struct vfsmount *);
ipc/mqueue.c +1 -1
index c595bed7bfcb..a8c2465ac0cb 100644
@@ -1554,7 +1554,7 @@ void mq_clear_sbinfo(struct ipc_namespace *ns)
 
 void mq_put_mnt(struct ipc_namespace *ns)
 {
-	kern_unmount(ns->mq_mnt);
+	kern_unmount_async(ns->mq_mnt);
 }
 
 static int __init init_mqueue_fs(void)
Signature · 1 line
2.20.1.611.gfbb209baf1-goog
Reviewed-by on v1 1/1
#1 lore raw
Quoted 52 lines · Salman Qazi

Prior to this patch, the kernel can spend a lot of time with this stack trace:

[<ffffffffbe5491e3>] __wait_rcu_gp+0x93/0xe0
[<ffffffffbe549418>] synchronize_sched+0x48/0x60
[<ffffffffbe7ae5b3>] kern_unmount+0x3a/0x46
[<ffffffffbe847c02>] mq_put_mnt+0x15/0x17
[<ffffffffbe8481af>] put_ipc_ns+0x36/0x8b

This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.

Since mntput() sleeps, it needs to be deferred to a work queue.

Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.

Tested: Ran the following program:

    int main(void)
    {
            int pid;
            int status;
            int i;

            for (i = 0; i < 1000; i++) {
                    pid = fork();
                    if (!pid) {
                            assert(!unshare(CLONE_NEWUSER|
                                      CLONE_NEWIPC|CLONE_NEWNS));
                            return 0;
                    }

                    assert(waitpid(pid, &status, 0) == pid);
            }
    }

Before:

$ time ./unshare2
real    0m9.784s
user    0m0.428s
sys     0m0.000s

After:

Quoted 7 lines
$ time ./unshare2
real    0m0.368s
user    0m0.226s
sys     0m0.122s
Quoted 86 lines
---
 fs/namespace.c     | 41 +++++++++++++++++++++++++++++++++++++++++
 include/linux/fs.h |  1 +
 ipc/mqueue.c       |  2 +-
 3 files changed, 43 insertions(+), 1 deletion(-)
fs/namespace.c +41 -0
index a677b59efd74..caa51ca81605 100644
@@ -3323,6 +3323,47 @@ void kern_unmount(struct vfsmount *mnt)
 }
 EXPORT_SYMBOL(kern_unmount);
 
+struct async_unmount_cb {
+       struct vfsmount *mnt;
+       struct work_struct work;
+       struct rcu_head rcu_head;
+};
+
+static void kern_unmount_work(struct work_struct *work)
+{
+       struct async_unmount_cb *cb = container_of(work,
+                       struct async_unmount_cb, work);
+
+       mntput(cb->mnt);
+       kfree(cb);
+}
+
+static void kern_unmount_rcu_cb(struct rcu_head *rcu_head)
+{
+       struct async_unmount_cb *cb = container_of(rcu_head,
+                       struct async_unmount_cb, rcu_head);
+
+       INIT_WORK(&cb->work, kern_unmount_work);
+       schedule_work(&cb->work);
+
+}
+
+void kern_unmount_async(struct vfsmount *mnt)
+{
+       /* release long term mount so mount point can be released */
+       if (!IS_ERR_OR_NULL(mnt)) {
+               struct async_unmount_cb *cb = kmalloc(sizeof(*cb), GFP_KERNEL);
+
+               if (cb) {
+                       real_mount(mnt)->mnt_ns = NULL;
+                       cb->mnt = mnt;
+                       call_rcu(&cb->rcu_head, kern_unmount_rcu_cb);
+               } else {
+                       kern_unmount(mnt);
+               }
+       }
+}
+
 bool our_mnt(struct vfsmount *mnt)
 {
        return check_mnt(real_mount(mnt));
include/linux/fs.h +1 -0
index 29d8e2cfed0e..8865997a8722 100644
@@ -2274,6 +2274,7 @@ extern int register_filesystem(struct file_system_type *);
 extern int unregister_filesystem(struct file_system_type *);
 extern struct vfsmount *kern_mount_data(struct file_system_type *, void *data);
 #define kern_mount(type) kern_mount_data(type, NULL)
+extern void kern_unmount_async(struct vfsmount *mnt);
 extern void kern_unmount(struct vfsmount *mnt);
 extern int may_umount_tree(struct vfsmount *);
 extern int may_umount(struct vfsmount *);
ipc/mqueue.c +1 -2
index c595bed7bfcb..a8c2465ac0cb 100644
@@ -1554,7 +1554,7 @@ void mq_clear_sbinfo(struct ipc_namespace *ns)
 
 void mq_put_mnt(struct ipc_namespace *ns)
 {
-       kern_unmount(ns->mq_mnt);
+       kern_unmount_async(ns->mq_mnt);
 }
 
 static int __init init_mqueue_fs(void)
--

2.20.1.611.gfbb209baf1-goog

maintainer
#2 lore raw
In reply to Salman Qazi
Quoted 11 lines · Salman Qazi

This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.

Since mntput() sleeps, it needs to be deferred to a work queue.

Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.

Ugh... I really doubt that it's correct. The caller is

                mq_put_mnt(ns);
                free_ipc_ns(ns);

and we have
static void mqueue_evict_inode(struct inode *inode) {

...

        ipc_ns = get_ns_from_inode(inode);

with

static struct ipc_namespace *get_ns_from_inode(struct inode *inode)

{
        struct ipc_namespace *ns;

        spin_lock(&mq_lock);
        ns = __get_ns_from_inode(inode);
        spin_unlock(&mq_lock);
        return ns;
}

and

static inline struct ipc_namespace *__get_ns_from_inode(struct inode *inode)

{
        return get_ipc_ns(inode->i_sb->s_fs_info);
}

with ->s_fs_info being the ipc_namespace we are freeing after mq_put_ns()

Are you saying that get_ipc_ns() after free_ipc_ns() is safe? Because ->evict_inode() *IS* called on umount. What happens to your patch if there was a regular file left on that filesystem?

Smells like a memory corruptor...

#3 lore raw
Quoted 45 lines · Al Viro
Quoted 11 lines · Salman Qazi

This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.

Since mntput() sleeps, it needs to be deferred to a work queue.

Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.

Ugh... I really doubt that it's correct. The caller is

                mq_put_mnt(ns);
                free_ipc_ns(ns);

and we have
static void mqueue_evict_inode(struct inode *inode) {

...

        ipc_ns = get_ns_from_inode(inode);

with

static struct ipc_namespace *get_ns_from_inode(struct inode *inode)

{
        struct ipc_namespace *ns;

        spin_lock(&mq_lock);
        ns = __get_ns_from_inode(inode);
        spin_unlock(&mq_lock);
        return ns;
}

and

static inline struct ipc_namespace *__get_ns_from_inode(struct inode *inode)

{
        return get_ipc_ns(inode->i_sb->s_fs_info);
}
Quoted 7 lines

with ->s_fs_info being the ipc_namespace we are freeing after mq_put_ns()

Are you saying that get_ipc_ns() after free_ipc_ns() is safe? Because ->evict_inode() *IS* called on umount. What happens to your patch if there was a regular file left on that filesystem?

Smells like a memory corruptor...

Actually, the full context in the caller is

        if (refcount_dec_and_lock(&ns->count, &mq_lock)) {
                mq_clear_sbinfo(ns);
                spin_unlock(&mq_lock);
                mq_put_mnt(ns);
                free_ipc_ns(ns);
        }

And

void mq_clear_sbinfo(struct ipc_namespace *ns)

{
        ns->mq_mnt->mnt_sb->s_fs_info = NULL;
}

Therefore, s_fs_info should be NULL before we proceed to unmount. So, as far as I know, it should not be possible to find the ipc_namespace from the mount.

#4 lore raw

Do you have any additional concerns?

Quoted 74 lines · Salman Qazi
Quoted 45 lines · Al Viro
Quoted 11 lines · Salman Qazi

This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.

Since mntput() sleeps, it needs to be deferred to a work queue.

Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.

Ugh... I really doubt that it's correct. The caller is

                mq_put_mnt(ns);
                free_ipc_ns(ns);

and we have
static void mqueue_evict_inode(struct inode *inode) {

...

        ipc_ns = get_ns_from_inode(inode);

with

static struct ipc_namespace *get_ns_from_inode(struct inode *inode)

{
        struct ipc_namespace *ns;

        spin_lock(&mq_lock);
        ns = __get_ns_from_inode(inode);
        spin_unlock(&mq_lock);
        return ns;
}

and

static inline struct ipc_namespace *__get_ns_from_inode(struct inode *inode)

{
        return get_ipc_ns(inode->i_sb->s_fs_info);
}
Quoted 7 lines

with ->s_fs_info being the ipc_namespace we are freeing after mq_put_ns()

Are you saying that get_ipc_ns() after free_ipc_ns() is safe? Because ->evict_inode() *IS* called on umount. What happens to your patch if there was a regular file left on that filesystem?

Smells like a memory corruptor...

Actually, the full context in the caller is

        if (refcount_dec_and_lock(&ns->count, &mq_lock)) {
                mq_clear_sbinfo(ns);
                spin_unlock(&mq_lock);
                mq_put_mnt(ns);
                free_ipc_ns(ns);
        }

And

void mq_clear_sbinfo(struct ipc_namespace *ns)

{
        ns->mq_mnt->mnt_sb->s_fs_info = NULL;
}

Therefore, s_fs_info should be NULL before we proceed to unmount. So, as far as I know, it should not be possible to find the ipc_namespace from the mount.

#5 lore raw

PATCH 1/1 fs, ipc: Use an asynchronous version of kern_unmount in IPC

In reply to Salman Qazi

Prior to this patch, the kernel can spend a lot of time with this stack trace:

[<ffffffffbe5491e3>] __wait_rcu_gp+0x93/0xe0
[<ffffffffbe549418>] synchronize_sched+0x48/0x60
[<ffffffffbe7ae5b3>] kern_unmount+0x3a/0x46
[<ffffffffbe847c02>] mq_put_mnt+0x15/0x17
[<ffffffffbe8481af>] put_ipc_ns+0x36/0x8b

This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.

Since mntput() sleeps, it needs to be deferred to a work queue.

Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.

Tested: Ran the following program:

    int main(void)
    {
            int pid;
            int status;
            int i;

            for (i = 0; i < 1000; i++) {
                    pid = fork();
                    if (!pid) {
                            assert(!unshare(CLONE_NEWUSER|
                                      CLONE_NEWIPC|CLONE_NEWNS));
                            return 0;
                    }

                    assert(waitpid(pid, &status, 0) == pid);
            }
    }

Before:

$ time ./unshare2
real    0m9.784s
user    0m0.428s
sys     0m0.000s

After:

$ time ./unshare2
real    0m0.368s
user    0m0.226s
sys     0m0.122s
---
 fs/namespace.c     | 41 +++++++++++++++++++++++++++++++++++++++++
 include/linux/fs.h |  1 +
 ipc/mqueue.c       |  2 +-
 3 files changed, 43 insertions(+), 1 deletion(-)
fs/namespace.c +41 -0
index 678ef175d63a..e60b473c3bbc 100644
@@ -3321,6 +3321,47 @@ void kern_unmount(struct vfsmount *mnt)
 }
 EXPORT_SYMBOL(kern_unmount);
 
+struct async_unmount_cb {
+	struct vfsmount *mnt;
+	struct work_struct work;
+	struct rcu_head rcu_head;
+};
+
+static void kern_unmount_work(struct work_struct *work)
+{
+	struct async_unmount_cb *cb = container_of(work,
+			struct async_unmount_cb, work);
+
+	mntput(cb->mnt);
+	kfree(cb);
+}
+
+static void kern_unmount_rcu_cb(struct rcu_head *rcu_head)
+{
+	struct async_unmount_cb *cb = container_of(rcu_head,
+			struct async_unmount_cb, rcu_head);
+
+	INIT_WORK(&cb->work, kern_unmount_work);
+	schedule_work(&cb->work);
+
+}
+
+void kern_unmount_async(struct vfsmount *mnt)
+{
+	/* release long term mount so mount point can be released */
+	if (!IS_ERR_OR_NULL(mnt)) {
+		struct async_unmount_cb *cb = kmalloc(sizeof(*cb), GFP_KERNEL);
+
+		if (cb) {
+			real_mount(mnt)->mnt_ns = NULL;
+			cb->mnt = mnt;
+			call_rcu(&cb->rcu_head, kern_unmount_rcu_cb);
+		} else {
+			kern_unmount(mnt);
+		}
+	}
+}
+
 bool our_mnt(struct vfsmount *mnt)
 {
 	return check_mnt(real_mount(mnt));
include/linux/fs.h +1 -0
index 29d8e2cfed0e..8865997a8722 100644
@@ -2274,6 +2274,7 @@ extern int register_filesystem(struct file_system_type *);
 extern int unregister_filesystem(struct file_system_type *);
 extern struct vfsmount *kern_mount_data(struct file_system_type *, void *data);
 #define kern_mount(type) kern_mount_data(type, NULL)
+extern void kern_unmount_async(struct vfsmount *mnt);
 extern void kern_unmount(struct vfsmount *mnt);
 extern int may_umount_tree(struct vfsmount *);
 extern int may_umount(struct vfsmount *);
ipc/mqueue.c +1 -1
index c595bed7bfcb..a8c2465ac0cb 100644
@@ -1554,7 +1554,7 @@ void mq_clear_sbinfo(struct ipc_namespace *ns)
 
 void mq_put_mnt(struct ipc_namespace *ns)
 {
-	kern_unmount(ns->mq_mnt);
+	kern_unmount_async(ns->mq_mnt);
 }
 
 static int __init init_mqueue_fs(void)
Signature · 1 line
2.21.0.352.gf09ad66450-goog