ipc,namespace: fix free vs allocation race
v3 cover ipc,namespace: fix free vs allocation race
The IPC namespace code frees ipc_namespace structures asynchronously, via a work queue item. This results in ipc_namespace structures being freed very slowly, and the allocation path getting false failures since the to-be-freed ipc_namespace structures have not been freed yet.
Fix that by having the allocator wait when there are ipc_namespace structures pending to be freed.
Also speed up the freeing of ipc_namespace structures. We had some discussions about this last year, and ended up trying out various "nicer" ideas that did not work, so I went back to the original, with Al Viro's suggestion for a helper function:
https://lore.kernel.org/all/[email protected]/
This series fixes both the false allocation failures, and the slow freeing of ipc_namespace structures.
v3: remove mq_put_mnt (thank you Giuseppe)
v2: a few more fs/namespace.c cleanups suggested by Al Viro (thank you!)
PATCH v3 1/2 ipc,namespace: make ipc namespace allocation wait for pending free
Currently the ipc namespace allocation will fail when there are ipc_namespace structures pending to be freed. This results in the simple test case below, as well as some real world workloads, to get allocation failures even when the number of ipc namespaces in actual use is way below the limit.
int main()
{
int i;
for (i = 0; i < 100000; i++) {
if (unshare(CLONE_NEWIPC) < 0)
error(EXIT_FAILURE, errno, "unshare");
}
}Make the allocation of an ipc_namespace wait for pending frees, so it will succeed.
real 6m19.197s
user 0m0.041s
sys 0m1.019s
--- ipc/namespace.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-)
ipc/namespace.c +16 -6
@@ -19,6 +19,12 @@#include "util.h" +/* +* The work queue is used to avoid the cost of synchronize_rcu in kern_unmount. +*/ +static void free_ipc (struct work_struct *unused); +static DECLARE_WORK(free_ipc_work, free_ipc); +static struct ucounts *inc_ipc_namespaces (struct user_namespace *ns) {return inc_ucount (ns,current_euid (),UCOUNT_IPC_NAMESPACES );
@@ -37,9 +43,18 @@ static struct ipc_namespace *create_ipc_ns(struct user_namespace *user_ns,int err; err = -ENOSPC ; + again: ucounts =inc_ipc_namespaces (user_ns); -if (!ucounts) +if (!ucounts) { +/* +* IPC namespaces are freed asynchronously, by free_ipc_work. +* If frees were pending, flush_work will wait, and +* return true. Fail the allocation if no frees are pending. +*/ +if (flush_work (&free_ipc_work)) + goto again; goto fail; + } err = -ENOMEM ; ns =kzalloc (sizeof (struct ipc_namespace ),GFP_KERNEL_ACCOUNT );
@@ -157,11 +172,6 @@ static void free_ipc(struct work_struct *unused)free_ipc_ns (n); } -/* -* The work queue is used to avoid the cost of synchronize_rcu in kern_unmount. -*/ -static DECLARE_WORK(free_ipc_work, free_ipc); -/* * put_ipc_ns - drop a reference to an ipc namespace. * @ns: the namespace to put
Signature · 1 line
2.38.1
PATCH v3 2/2 ipc,namespace: batch free ipc_namespace structures
In reply to Rik van RielInstead of waiting for an RCU grace period between each ipc_namespace structure that is being freed, wait an RCU grace period for every batch of ipc_namespace structures.
Thanks to Al Viro for the suggestion of the helper function.
This speeds up the run time of the test case that allocates ipc_namespaces in a loop from 6 minutes, to a little over 1 second:
real 0m1.192s
user 0m0.038s
sys 0m1.152s
--- fs/namespace.c | 18 ++++++++++++++---- include/linux/mount.h | 1 + ipc/mqueue.c | 5 ----- ipc/namespace.c | 13 ++++++++++--- ipc/util.h | 2 -- 5 files changed, 25 insertions(+), 14 deletions(-)
fs/namespace.c +14 -4
@@ -1397,6 +1397,17 @@ struct vfsmount *mntget(struct vfsmount *mnt) }EXPORT_SYMBOL (mntget); +/* +* Make a mount point inaccessible to new lookups. +* Because there may still be current users, the caller MUST WAIT +* for an RCU grace period before destroying the mount point. +*/ +void mnt_make_shortterm (struct vfsmount *mnt) +{ +if (mnt) +real_mount (mnt)->mnt_ns =NULL ; +} +/** * path_is_mountpoint() - Check if path is a mount in the current namespace. * @path: path to check
@@ -4573,8 +4584,8 @@ EXPORT_SYMBOL_GPL(kern_mount);void kern_unmount (struct vfsmount *mnt) {/* release long term mount so mount point can be released */ -if (!IS_ERR_OR_NULL (mnt)) { -real_mount (mnt)->mnt_ns =NULL ; +if (!IS_ERR (mnt)) { +mnt_make_shortterm (mnt);synchronize_rcu ();/* yecchhh... */ mntput (mnt); }
@@ -4586,8 +4597,7 @@ void kern_unmount_array(struct vfsmount *mnt[], unsigned int num)unsigned int i;for (i =0 ; i < num; i++) -if (mnt[i]) -real_mount (mnt[i])->mnt_ns =NULL ; +mnt_make_shortterm (mnt[i]);synchronize_rcu_expedited ();for (i =0 ; i < num; i++)mntput (mnt[i]);
include/linux/mount.h +1 -0
@@ -88,6 +88,7 @@ extern void mnt_drop_write(struct vfsmount *mnt);extern void mnt_drop_write_file (struct file *file);extern void mntput (struct vfsmount *mnt);extern struct vfsmount *mntget (struct vfsmount *mnt); +extern void mnt_make_shortterm (struct vfsmount *mnt);extern struct vfsmount *mnt_clone_internal (const struct path *path);extern bool __mnt_is_readonly (struct vfsmount *mnt);extern bool mnt_may_suid (struct vfsmount *mnt);
ipc/mqueue.c +0 -5
@@ -1709,11 +1709,6 @@ void mq_clear_sbinfo(struct ipc_namespace *ns) ns->mq_mnt->mnt_sb->s_fs_info =NULL ; } -void mq_put_mnt (struct ipc_namespace *ns) -{ -kern_unmount (ns->mq_mnt); -} -static int __init init_mqueue_fs (void ) {int error;
ipc/namespace.c +10 -3
@@ -145,10 +145,11 @@ void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,static void free_ipc_ns (struct ipc_namespace *ns) { -/* mq_put_mnt() waits for a grace period as kern_unmount() -* uses synchronize_rcu(). +/* +* Caller needs to wait for an RCU grace period to have passed +* after making the mount point inaccessible to new accesses. */ -mq_put_mnt (ns); +mntput (ns->mq_mnt);sem_exit_ns (ns);msg_exit_ns (ns);shm_exit_ns (ns);
@@ -168,6 +169,12 @@ static void free_ipc(struct work_struct *unused)struct llist_node *node =llist_del_all (&free_ipc_list);struct ipc_namespace *n, *t; +llist_for_each_entry_safe (n, t, node, mnt_llist) +mnt_make_shortterm (n->mq_mnt); + +/* Wait for any last users to have gone away. */ +synchronize_rcu (); +llist_for_each_entry_safe (n, t, node, mnt_llist)free_ipc_ns (n); }
ipc/util.h +0 -2
@@ -56,10 +56,8 @@ struct pid_namespace;#ifdef CONFIG_POSIX_MQUEUE extern void mq_clear_sbinfo (struct ipc_namespace *ns); -extern void mq_put_mnt (struct ipc_namespace *ns);#else static inline void mq_clear_sbinfo (struct ipc_namespace *ns) { } -static inline void mq_put_mnt (struct ipc_namespace *ns) { }#endif #ifdef CONFIG_SYSVIPC
Signature · 1 line
2.38.1
Quoted 14 lines · Rik van Riel
The IPC namespace code frees
ipc_namespacestructures asynchronously, via a work queue item. This results inipc_namespacestructures being freed very slowly, and the allocation path getting false failures since the to-be-freedipc_namespacestructures have not been freed yet.Fix that by having the allocator wait when there are
ipc_namespacestructures pending to be freed.Also speed up the freeing of
ipc_namespacestructures. We had some discussions about this last year, and ended up trying out various "nicer" ideas that did not work, so I went back to the original, with Al Viro's suggestion for a helper function:
Quoted 7 lines
https://lore.kernel.org/all/[email protected]/
This series fixes both the false allocation failures, and the slow freeing of
ipc_namespacestructures.v3: remove
mq_put_mnt(thank you Giuseppe)
v2: a few morefs/namespace.ccleanups suggested by Al Viro (thank you!)
Al Viro
maintainerQuoted 14 lines · Rik van Riel
The IPC namespace code frees
ipc_namespacestructures asynchronously, via a work queue item. This results inipc_namespacestructures being freed very slowly, and the allocation path getting false failures since the to-be-freedipc_namespacestructures have not been freed yet.Fix that by having the allocator wait when there are
ipc_namespacestructures pending to be freed.Also speed up the freeing of
ipc_namespacestructures. We had some discussions about this last year, and ended up trying out various "nicer" ideas that did not work, so I went back to the original, with Al Viro's suggestion for a helper function:
Quoted 7 lines
https://lore.kernel.org/all/[email protected]/
This series fixes both the false allocation failures, and the slow freeing of
ipc_namespacestructures.v3: remove
mq_put_mnt(thank you Giuseppe)
v2: a few morefs/namespace.ccleanups suggested by Al Viro (thank you!)
Applied (#work.namespace)