fs, ipc: Use an asynchronous version of kern_unmount in IPC
PATCH 1/1 fs, ipc: Use an asynchronous version of kern_unmount in IPC
Prior to this patch, the kernel can spend a lot of time with this stack trace:
[<ffffffffbe5491e3>] __wait_rcu_gp+0x93/0xe0 [<ffffffffbe549418>] synchronize_sched+0x48/0x60 [<ffffffffbe7ae5b3>] kern_unmount+0x3a/0x46 [<ffffffffbe847c02>] mq_put_mnt+0x15/0x17 [<ffffffffbe8481af>] put_ipc_ns+0x36/0x8b
This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.
Since mntput() sleeps, it needs to be deferred to a work queue.
Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.
Tested: Ran the following program:
int main(void)
{
int pid;
int status;
int i;
for (i = 0; i < 1000; i++) {
pid = fork();
if (!pid) {
assert(!unshare(CLONE_NEWUSER|
CLONE_NEWIPC|CLONE_NEWNS));
return 0;
}
assert(waitpid(pid, &status, 0) == pid);
}
}Before:
$ time ./unshare2
real 0m9.784s user 0m0.428s sys 0m0.000s
After:
$ time ./unshare2
real 0m0.368s user 0m0.226s sys 0m0.122s
--- fs/namespace.c | 41 +++++++++++++++++++++++++++++++++++++++++ include/linux/fs.h | 1 + ipc/mqueue.c | 2 +- 3 files changed, 43 insertions(+), 1 deletion(-)
fs/namespace.c +41 -0
@@ -3323,6 +3323,47 @@ void kern_unmount(struct vfsmount *mnt) }EXPORT_SYMBOL (kern_unmount); +struct async_unmount_cb { +struct vfsmount *mnt; +struct work_struct work; +struct rcu_head rcu_head; +}; + +static void kern_unmount_work (struct work_struct *work) +{ +struct async_unmount_cb *cb =container_of (work, + struct async_unmount_cb, work); + +mntput (cb->mnt); +kfree (cb); +} + +static void kern_unmount_rcu_cb (struct rcu_head *rcu_head) +{ +struct async_unmount_cb *cb =container_of (rcu_head, + struct async_unmount_cb, rcu_head); + +INIT_WORK (&cb->work, kern_unmount_work); +schedule_work (&cb->work); + +} + +void kern_unmount_async (struct vfsmount *mnt) +{ +/* release long term mount so mount point can be released */ +if (!IS_ERR_OR_NULL (mnt)) { +struct async_unmount_cb *cb =kmalloc (sizeof (*cb),GFP_KERNEL ); + +if (cb) { +real_mount (mnt)->mnt_ns =NULL ; + cb->mnt = mnt; +call_rcu (&cb->rcu_head, kern_unmount_rcu_cb); + }else { +kern_unmount (mnt); + } + } +} +bool our_mnt (struct vfsmount *mnt) {return check_mnt (real_mount (mnt));
include/linux/fs.h +1 -0
@@ -2274,6 +2274,7 @@ extern int register_filesystem(struct file_system_type *);extern int unregister_filesystem (struct file_system_type *);extern struct vfsmount *kern_mount_data (struct file_system_type *,void *data);#define kern_mount (type) kern_mount_data(type, NULL) +extern void kern_unmount_async (struct vfsmount *mnt);extern void kern_unmount (struct vfsmount *mnt);extern int may_umount_tree (struct vfsmount *);extern int may_umount (struct vfsmount *);
ipc/mqueue.c +1 -1
@@ -1554,7 +1554,7 @@ void mq_clear_sbinfo(struct ipc_namespace *ns)void mq_put_mnt (struct ipc_namespace *ns) { -kern_unmount (ns->mq_mnt); +kern_unmount_async (ns->mq_mnt); }static int __init init_mqueue_fs (void )
Signature · 1 line
2.20.1.611.gfbb209baf1-goog
Quoted 52 lines · Salman Qazi
Prior to this patch, the kernel can spend a lot of time with this stack trace:
[<ffffffffbe5491e3>] __wait_rcu_gp+0x93/0xe0 [<ffffffffbe549418>] synchronize_sched+0x48/0x60 [<ffffffffbe7ae5b3>] kern_unmount+0x3a/0x46 [<ffffffffbe847c02>] mq_put_mnt+0x15/0x17 [<ffffffffbe8481af>] put_ipc_ns+0x36/0x8bThis patch solves the issue by removing
synchronize_rcufrommq_put_mnt. This is done by implementing an asynchronous version ofkern_unmount.Since
mntput()sleeps, it needs to be deferred to a work queue.Additionally, the callers of
mq_put_mntappear to be safe having it behave asynchronously. In particular,put_ipc_nscallsmq_clear_sbinfowhich renders the inode inaccessible for the purposes ofmqueue_createby makings_fs_infoNULL. This appears
to be the thing that prevents access whilefree_ipc_nsis taking place. So, the unmount should be able to proceed lazily.Tested: Ran the following program:
int main(void) { int pid; int status; int i; for (i = 0; i < 1000; i++) { pid = fork(); if (!pid) { assert(!unshare(CLONE_NEWUSER| CLONE_NEWIPC|CLONE_NEWNS)); return 0; } assert(waitpid(pid, &status, 0) == pid); } }Before:
$ time ./unshare2real 0m9.784s user 0m0.428s sys 0m0.000sAfter:
Quoted 7 lines
$ time ./unshare2real 0m0.368s user 0m0.226s sys 0m0.122sSigned-off-by Salman Qazi <[email protected]>
Quoted 86 lines
--- fs/namespace.c | 41 +++++++++++++++++++++++++++++++++++++++++ include/linux/fs.h | 1 + ipc/mqueue.c | 2 +- 3 files changed, 43 insertions(+), 1 deletion(-)fs/namespace.c +41
-0@@ -3323,6 +3323,47 @@ void kern_unmount(struct vfsmount *mnt) }EXPORT_SYMBOL (kern_unmount); +struct async_unmount_cb { +struct vfsmount *mnt; +struct work_struct work; +struct rcu_head rcu_head; +}; + +static void kern_unmount_work (struct work_struct *work) +{ +struct async_unmount_cb *cb =container_of (work, + struct async_unmount_cb, work); + +mntput (cb->mnt); +kfree (cb); +} + +static void kern_unmount_rcu_cb (struct rcu_head *rcu_head) +{ +struct async_unmount_cb *cb =container_of (rcu_head, + struct async_unmount_cb, rcu_head); + +INIT_WORK (&cb->work, kern_unmount_work); +schedule_work (&cb->work); + +} + +void kern_unmount_async (struct vfsmount *mnt) +{ +/* release long term mount so mount point can be released */ +if (!IS_ERR_OR_NULL (mnt)) { +struct async_unmount_cb *cb =kmalloc (sizeof (*cb),GFP_KERNEL ); + +if (cb) { +real_mount (mnt)->mnt_ns =NULL ; + cb->mnt = mnt; +call_rcu (&cb->rcu_head, kern_unmount_rcu_cb); + }else { +kern_unmount (mnt); + } + } +} +bool our_mnt (struct vfsmount *mnt) {return check_mnt (real_mount (mnt));include/linux/fs.h +1
-0@@ -2274,6 +2274,7 @@ extern int register_filesystem(struct file_system_type *);extern int unregister_filesystem (struct file_system_type *);extern struct vfsmount *kern_mount_data (struct file_system_type *,void *data);#define kern_mount (type) kern_mount_data(type, NULL) +extern void kern_unmount_async (struct vfsmount *mnt);extern void kern_unmount (struct vfsmount *mnt);extern int may_umount_tree (struct vfsmount *);extern int may_umount (struct vfsmount *);ipc/mqueue.c +1
-2@@ -1554,7 +1554,7 @@ void mq_clear_sbinfo(struct ipc_namespace *ns)void mq_put_mnt (struct ipc_namespace *ns) { -kern_unmount (ns->mq_mnt); +kern_unmount_async (ns->mq_mnt); }static int __init init_mqueue_fs (void ) --2.20.1.611.gfbb209baf1-goog
Al Viro
maintainerQuoted 11 lines · Salman Qazi
This patch solves the issue by removing
synchronize_rcufrommq_put_mnt. This is done by implementing an asynchronous version ofkern_unmount.Since
mntput()sleeps, it needs to be deferred to a work queue.Additionally, the callers of
mq_put_mntappear to be safe having it behave asynchronously. In particular,put_ipc_nscallsmq_clear_sbinfowhich renders the inode inaccessible for the purposes ofmqueue_createby makings_fs_infoNULL. This appears
to be the thing that prevents access whilefree_ipc_nsis taking place. So, the unmount should be able to proceed lazily.
Ugh... I really doubt that it's correct. The caller is
mq_put_mnt(ns);
free_ipc_ns(ns);and we have
static void mqueue_evict_inode(struct inode *inode) {
...
ipc_ns = get_ns_from_inode(inode);
with
static struct ipc_namespace *get_ns_from_inode(struct inode *inode)
{
struct ipc_namespace *ns;
spin_lock(&mq_lock);
ns = __get_ns_from_inode(inode);
spin_unlock(&mq_lock);
return ns;
}and
static inline struct ipc_namespace *__get_ns_from_inode(struct inode *inode)
{
return get_ipc_ns(inode->i_sb->s_fs_info);
}with ->s_fs_info being the ipc_namespace we are freeing after mq_put_ns()
Are you saying that get_ipc_ns() after free_ipc_ns() is safe? Because ->evict_inode() *IS* called on umount. What happens to your patch if there was a regular file left on that filesystem?
Smells like a memory corruptor...
Quoted 45 lines · Al Viro
Quoted 11 lines · Salman Qazi
This patch solves the issue by removing
synchronize_rcufrommq_put_mnt. This is done by implementing an asynchronous version ofkern_unmount.Since
mntput()sleeps, it needs to be deferred to a work queue.Additionally, the callers of
mq_put_mntappear to be safe having it behave asynchronously. In particular,put_ipc_nscallsmq_clear_sbinfowhich renders the inode inaccessible for the purposes ofmqueue_createby makings_fs_infoNULL. This appears
to be the thing that prevents access whilefree_ipc_nsis taking place. So, the unmount should be able to proceed lazily.Ugh... I really doubt that it's correct. The caller is
mq_put_mnt(ns); free_ipc_ns(ns);and we have
static voidmqueue_evict_inode(struct inode *inode){...
ipc_ns = get_ns_from_inode(inode);with
static struct
ipc_namespace*get_ns_from_inode(struct inode *inode){ struct ipc_namespace *ns; spin_lock(&mq_lock); ns = __get_ns_from_inode(inode); spin_unlock(&mq_lock); return ns; }and
static inline struct
ipc_namespace*__get_ns_from_inode(struct inode *inode){ return get_ipc_ns(inode->i_sb->s_fs_info); }
Quoted 7 lines
with ->s_fs_info being the
ipc_namespacewe are freeing aftermq_put_ns()Are you saying that
get_ipc_ns()afterfree_ipc_ns()is safe? Because ->evict_inode() *IS* called on umount. What happens to your patch if there was a regular file left on that filesystem?Smells like a memory corruptor...
Actually, the full context in the caller is
if (refcount_dec_and_lock(&ns->count, &mq_lock)) {
mq_clear_sbinfo(ns);
spin_unlock(&mq_lock);
mq_put_mnt(ns);
free_ipc_ns(ns);
}And
void mq_clear_sbinfo(struct ipc_namespace *ns)
{
ns->mq_mnt->mnt_sb->s_fs_info = NULL;
}Therefore, s_fs_info should be NULL before we proceed to unmount. So, as far as I know, it should not be possible to find the ipc_namespace from the mount.
Do you have any additional concerns?
Quoted 74 lines · Salman Qazi
Quoted 45 lines · Al Viro
Quoted 11 lines · Salman Qazi
This patch solves the issue by removing
synchronize_rcufrommq_put_mnt. This is done by implementing an asynchronous version ofkern_unmount.Since
mntput()sleeps, it needs to be deferred to a work queue.Additionally, the callers of
mq_put_mntappear to be safe having it behave asynchronously. In particular,put_ipc_nscallsmq_clear_sbinfowhich renders the inode inaccessible for the purposes ofmqueue_createby makings_fs_infoNULL. This appears
to be the thing that prevents access whilefree_ipc_nsis taking place. So, the unmount should be able to proceed lazily.Ugh... I really doubt that it's correct. The caller is
mq_put_mnt(ns); free_ipc_ns(ns);and we have
static voidmqueue_evict_inode(struct inode *inode){...
ipc_ns = get_ns_from_inode(inode);with
static struct
ipc_namespace*get_ns_from_inode(struct inode *inode){ struct ipc_namespace *ns; spin_lock(&mq_lock); ns = __get_ns_from_inode(inode); spin_unlock(&mq_lock); return ns; }and
static inline struct
ipc_namespace*__get_ns_from_inode(struct inode *inode){ return get_ipc_ns(inode->i_sb->s_fs_info); }Quoted 7 lines
with ->s_fs_info being the
ipc_namespacewe are freeing aftermq_put_ns()Are you saying that
get_ipc_ns()afterfree_ipc_ns()is safe? Because ->evict_inode() *IS* called on umount. What happens to your patch if there was a regular file left on that filesystem?Smells like a memory corruptor...
Actually, the full context in the caller is
if (refcount_dec_and_lock(&ns->count, &mq_lock)) { mq_clear_sbinfo(ns); spin_unlock(&mq_lock); mq_put_mnt(ns); free_ipc_ns(ns); }And
void
mq_clear_sbinfo(struct ipc_namespace *ns){ ns->mq_mnt->mnt_sb->s_fs_info = NULL; }Therefore,
s_fs_infoshould be NULL before we proceed to unmount. So, as far as I know, it should not be possible to find theipc_namespacefrom the mount.
PATCH 1/1 fs, ipc: Use an asynchronous version of kern_unmount in IPC
In reply to Salman QaziPrior to this patch, the kernel can spend a lot of time with this stack trace:
[<ffffffffbe5491e3>] __wait_rcu_gp+0x93/0xe0 [<ffffffffbe549418>] synchronize_sched+0x48/0x60 [<ffffffffbe7ae5b3>] kern_unmount+0x3a/0x46 [<ffffffffbe847c02>] mq_put_mnt+0x15/0x17 [<ffffffffbe8481af>] put_ipc_ns+0x36/0x8b
This patch solves the issue by removing synchronize_rcu from mq_put_mnt. This is done by implementing an asynchronous version of kern_unmount.
Since mntput() sleeps, it needs to be deferred to a work queue.
Additionally, the callers of mq_put_mnt appear to be safe having it behave asynchronously. In particular, put_ipc_ns calls mq_clear_sbinfo which renders the inode inaccessible for the purposes of mqueue_create by making s_fs_info NULL. This appears
to be the thing that prevents access while free_ipc_ns is taking place. So, the unmount should be able to proceed lazily.
Tested: Ran the following program:
int main(void)
{
int pid;
int status;
int i;
for (i = 0; i < 1000; i++) {
pid = fork();
if (!pid) {
assert(!unshare(CLONE_NEWUSER|
CLONE_NEWIPC|CLONE_NEWNS));
return 0;
}
assert(waitpid(pid, &status, 0) == pid);
}
}Before:
$ time ./unshare2
real 0m9.784s user 0m0.428s sys 0m0.000s
After:
$ time ./unshare2
real 0m0.368s user 0m0.226s sys 0m0.122s
--- fs/namespace.c | 41 +++++++++++++++++++++++++++++++++++++++++ include/linux/fs.h | 1 + ipc/mqueue.c | 2 +- 3 files changed, 43 insertions(+), 1 deletion(-)
fs/namespace.c +41 -0
@@ -3321,6 +3321,47 @@ void kern_unmount(struct vfsmount *mnt) }EXPORT_SYMBOL (kern_unmount); +struct async_unmount_cb { +struct vfsmount *mnt; +struct work_struct work; +struct rcu_head rcu_head; +}; + +static void kern_unmount_work (struct work_struct *work) +{ +struct async_unmount_cb *cb =container_of (work, + struct async_unmount_cb, work); + +mntput (cb->mnt); +kfree (cb); +} + +static void kern_unmount_rcu_cb (struct rcu_head *rcu_head) +{ +struct async_unmount_cb *cb =container_of (rcu_head, + struct async_unmount_cb, rcu_head); + +INIT_WORK (&cb->work, kern_unmount_work); +schedule_work (&cb->work); + +} + +void kern_unmount_async (struct vfsmount *mnt) +{ +/* release long term mount so mount point can be released */ +if (!IS_ERR_OR_NULL (mnt)) { +struct async_unmount_cb *cb =kmalloc (sizeof (*cb),GFP_KERNEL ); + +if (cb) { +real_mount (mnt)->mnt_ns =NULL ; + cb->mnt = mnt; +call_rcu (&cb->rcu_head, kern_unmount_rcu_cb); + }else { +kern_unmount (mnt); + } + } +} +bool our_mnt (struct vfsmount *mnt) {return check_mnt (real_mount (mnt));
include/linux/fs.h +1 -0
@@ -2274,6 +2274,7 @@ extern int register_filesystem(struct file_system_type *);extern int unregister_filesystem (struct file_system_type *);extern struct vfsmount *kern_mount_data (struct file_system_type *,void *data);#define kern_mount (type) kern_mount_data(type, NULL) +extern void kern_unmount_async (struct vfsmount *mnt);extern void kern_unmount (struct vfsmount *mnt);extern int may_umount_tree (struct vfsmount *);extern int may_umount (struct vfsmount *);
ipc/mqueue.c +1 -1
@@ -1554,7 +1554,7 @@ void mq_clear_sbinfo(struct ipc_namespace *ns)void mq_put_mnt (struct ipc_namespace *ns) { -kern_unmount (ns->mq_mnt); +kern_unmount_async (ns->mq_mnt); }static int __init init_mqueue_fs (void )
Signature · 1 line
2.21.0.352.gf09ad66450-goog