posix mqueue namespace (v12)
cover posix mqueue namespace (v12)
Posix message queue support in ipc namespaces has had several submissions by various people. Hopefully this version, based on code by both Cedric Le Goater and Nadia Derbey has a shot at making it upstream.
To create a new ipc namespace with posix mq support, you would
unshare(CLONE_NEWIPC|CLONE_NEWNS); umount /dev/mqueue mount -t mqueue mqueue /dev/mqueue
It's perfectly valid to do vfs operations on files in another ipc_namespace's /dev/mqueue, but any use of mq_open(3) and friends will act in your own ipc_ns.
thanks,
-serge
PATCH 1/2 mqueue ns: move mqueue_mnt into struct ipc_namespace
Move mqueue vfsmount plus a few tunables into the ipc_namespace struct. The CONFIG_IPC_NS boolean and the ipc_namespace struct will serve both the posix message queue namespaces and the SYSV ipc namespaces.
--- include/linux/ipc_namespace.h | 32 ++++++++++-- init/Kconfig | 4 +- ipc/mqueue.c | 116 ++++++++++++++++++++++------------------- ipc/msgutil.c | 21 +++++++ ipc/namespace.c | 2 + ipc/util.c | 9 --- ipc/util.h | 15 +++++ 7 files changed, 131 insertions(+), 68 deletions(-)
include/linux/ipc_namespace.h +28 -4
@@ -44,24 +44,48 @@ struct ipc_namespace {int shm_tot;struct notifier_block ipcns_nb; + +struct vfsmount *mq_mnt; +unsigned int mq_queues_count; +unsigned int mq_queues_max; +unsigned int mq_msg_max; +unsigned int mq_msgsize_max; + };extern struct ipc_namespace init_ipc_ns;extern atomic_t nr_ipc_ns; -#ifdef CONFIG_SYSVIPC +#if defined(CONFIG_POSIX_MQUEUE ) || defined(CONFIG_SYSVIPC )#define INIT_IPC_NS (ns) .ns = &init_ipc_ns, +#else +#define INIT_IPC_NS (ns) +#endif +#ifdef CONFIG_SYSVIPC extern int register_ipcns_notifier (struct ipc_namespace *);extern int cond_register_ipcns_notifier (struct ipc_namespace *);extern void unregister_ipcns_notifier (struct ipc_namespace *);extern int ipcns_notify (unsigned long ); -#else /* CONFIG_SYSVIPC */ -#define INIT_IPC_NS (ns) +#define register_ipcns_notifier (ns) +#define cond_register_ipcns_notifier (ns) +#define unregister_ipcns_notifier (ns) +#define ipcns_notify (l)#endif /* CONFIG_SYSVIPC */ -#if defined(CONFIG_SYSVIPC ) && defined(CONFIG_IPC_NS ) +#ifdef CONFIG_POSIX_MQUEUE +extern void mq_init_ns (struct ipc_namespace *ns); +/* default values */ +#define DFLT_QUEUESMAX 256/* max number of message queues */ +#define DFLT_MSGMAX 10/* max number of messages in each queue */ +#define HARD_MSGMAX (131072/sizeof(void *)) +#define DFLT_MSGSIZEMAX 8192/* max message size */ +#else +#define mq_init_ns (ns) +#endif + +#if defined(CONFIG_IPC_NS )extern void free_ipc_ns (struct kref *kref);extern struct ipc_namespace *copy_ipcs (unsigned long flags,struct ipc_namespace *ns);
init/Kconfig +2 -2
@@ -489,10 +489,10 @@ config UTS_NS config IPC_NS bool "IPC namespace" - depends on NAMESPACES && SYSVIPC + depends on NAMESPACES && (SYSVIPC || POSIX_MQUEUE) help In this namespace tasks work with IPC ids which correspond to - different IPC objects in different namespaces + different IPC objects in different namespaces. config USER_NS bool "User namespace (EXPERIMENTAL)"
ipc/mqueue.c +63 -53
336 lines not on this page · raw message
ipc/msgutil.c +21 -0
@@ -13,10 +13,31 @@#include <linux/security.h> #include <linux/slab.h> #include <linux/ipc.h> +#include <linux/ipc_namespace.h> #include <asm/uaccess.h> #include "util.h" +/* +* The next 2 defines are here bc this is the only file +* compiled when either CONFIG_SYSVIPC and CONFIG_POSIX_MQUEUE +* and not CONFIG_IPC_NS. +*/ +struct ipc_namespace init_ipc_ns = { + .kref = { + .refcount =ATOMIC_INIT (2 ), + }, +#ifdef CONFIG_POSIX_MQUEUE + .mq_mnt =NULL , + .mq_queues_count =0 , + .mq_queues_max =DFLT_QUEUESMAX , + .mq_msg_max =DFLT_MSGMAX , + .mq_msgsize_max =DFLT_MSGSIZEMAX , +#endif +}; + +atomic_t nr_ipc_ns =ATOMIC_INIT (1 ); +struct msg_msgseg {struct msg_msgseg * next;/* the next part of the message follows immediately */
ipc/namespace.c +2 -0
@@ -25,6 +25,7 @@ static struct ipc_namespace *clone_ipc_ns(struct ipc_namespace *old_ns)sem_init_ns (ns);msg_init_ns (ns);shm_init_ns (ns); +mq_init_ns (ns);/* * msgmni has already been computed for the new ipc ns.
@@ -101,6 +102,7 @@ void free_ipc_ns(struct kref *kref)sem_exit_ns (ns);msg_exit_ns (ns);shm_exit_ns (ns); +mq_exit_ns (ns);kfree (ns);atomic_dec (&nr_ipc_ns);
ipc/util.c +0 -9
@@ -47,15 +47,6 @@ struct ipc_proc_iface {int (*show)(struct seq_file *,void *); }; -struct ipc_namespace init_ipc_ns = { - .kref = { - .refcount =ATOMIC_INIT (2 ), - }, -}; - -atomic_t nr_ipc_ns =ATOMIC_INIT (1 ); - -#ifdef CONFIG_MEMORY_HOTPLUG static void ipc_memory_notifier (struct work_struct *work)
ipc/util.h +15 -0
@@ -20,6 +20,13 @@ void shm_init (void);struct ipc_namespace ; +#ifdef CONFIG_POSIX_MQUEUE +void mq_exit_ns (struct ipc_namespace *ns); +#else +#define mq_exit_ns (ns) +#endif + +#ifdef CONFIG_SYSVIPC void sem_init_ns (struct ipc_namespace *ns);void msg_init_ns (struct ipc_namespace *ns);void shm_init_ns (struct ipc_namespace *ns);
@@ -27,6 +34,14 @@ void shm_init_ns(struct ipc_namespace *ns);void sem_exit_ns (struct ipc_namespace *ns);void msg_exit_ns (struct ipc_namespace *ns);void shm_exit_ns (struct ipc_namespace *ns); +#else +#define sem_init_ns (ns) +#define msg_init_ns (ns) +#define shm_init_ns (ns) +#define sem_exit_ns (ns) +#define msg_exit_ns (ns) +#define shm_exit_ns (ns) +#endif /* * Structure that holds the parameters needed by the ipc operations
Signature · 1 line
1.5.4.3
Re: [PATCH 1/2] mqueue ns: move mqueue_mnt into struct ipc_namespace
Quoted 60 lines · Serge E. Hallyn
Move mqueue vfsmount plus a few tunables into the
ipc_namespacestruct. TheCONFIG_IPC_NSboolean and theipc_namespacestruct will serve both the posix message queue namespaces and the SYSV ipc namespaces.Signed-off-by Cedric Le Goater <[email protected]>Signed-off-by Serge E. Hallyn <[email protected]>--- include/linux/ipc_namespace.h | 32 ++++++++++-- init/Kconfig | 4 +- ipc/mqueue.c | 116 ++++++++++++++++++++++------------------- ipc/msgutil.c | 21 +++++++ ipc/namespace.c | 2 + ipc/util.c | 9 --- ipc/util.h | 15 +++++ 7 files changed, 131 insertions(+), 68 deletions(-)include/linux/ipc_namespace.h +16
-4@@ -44,24 +44,48 @@ struct ipc_namespace {int shm_tot;struct notifier_block ipcns_nb; + +struct vfsmount *mq_mnt; +unsigned int mq_queues_count; +unsigned int mq_queues_max; +unsigned int mq_msg_max; +unsigned int mq_msgsize_max; + };extern struct ipc_namespace init_ipc_ns;extern atomic_t nr_ipc_ns; -#ifdef CONFIG_SYSVIPC +#if defined(CONFIG_POSIX_MQUEUE ) || defined(CONFIG_SYSVIPC )#define INIT_IPC_NS (ns) .ns = &init_ipc_ns, +#else +#define INIT_IPC_NS (ns) +#endif +#ifdef CONFIG_SYSVIPC extern int register_ipcns_notifier (struct ipc_namespace *);extern int cond_register_ipcns_notifier (struct ipc_namespace *);extern void unregister_ipcns_notifier (struct ipc_namespace *);extern int ipcns_notify (unsigned long ); -#else /* CONFIG_SYSVIPC */ -#define INIT_IPC_NS (ns) +#define register_ipcns_notifier (ns) +#define cond_register_ipcns_notifier (ns) +#define unregister_ipcns_notifier (ns) +#define ipcns_notify (l)#endif /* CONFIG_SYSVIPC */ -#if defined(CONFIG_SYSVIPC ) && defined(CONFIG_IPC_NS )
include/linux/ipc_namespace.h struct ipc_namespace { · line 86
Quoted 10 lines
include/linux/ipc_namespace.h +10
-0+#ifdef CONFIG_POSIX_MQUEUE +extern void mq_init_ns (struct ipc_namespace *ns); +/* default values */ +#define DFLT_QUEUESMAX 256/* max number of message queues */ +#define DFLT_MSGMAX 10/* max number of messages in each queue */ +#define HARD_MSGMAX (131072/sizeof(void *)) +#define DFLT_MSGSIZEMAX 8192/* max message size */ +#else +#define mq_init_ns (ns) +#endif
Do all these empty suckers need do{}while(0)'s?
Quoted 60 lines
include/linux/ipc_namespace.h +1
-0+#if defined(CONFIG_IPC_NS )extern void free_ipc_ns (struct kref *kref);extern struct ipc_namespace *copy_ipcs (unsigned long flags,struct ipc_namespace *ns);init/Kconfig +2
-2@@ -489,10 +489,10 @@ config UTS_NS config IPC_NS bool "IPC namespace" - depends on NAMESPACES && SYSVIPC + depends on NAMESPACES && (SYSVIPC || POSIX_MQUEUE) help In this namespace tasks work with IPC ids which correspond to - different IPC objects in different namespaces + different IPC objects in different namespaces. config USER_NS bool "User namespace (EXPERIMENTAL)"ipc/mqueue.c +1
-12@@ -31,6 +31,7 @@#include <linux/mutex.h> #include <linux/nsproxy.h> #include <linux/pid.h> +#include <linux/ipc_namespace.h> #include <net/sock.h> #include "util.h" @@ -46,12 +47,6 @@#define STATE_PENDING 1#define STATE_READY 2 -/* default values */ -#define DFLT_QUEUESMAX 256/* max number of message queues */ -#define DFLT_MSGMAX 10/* max number of messages in each queue */ -#define HARD_MSGMAX (131072/sizeof(void*)) -#define DFLT_MSGSIZEMAX 8192/* max message size */ -/* * Define the ranges various user-specified maximum values can * be set to. @@ -95,12 +90,6 @@ static void remove_notification(struct mqueue_inode_info *info);static spinlock_t mq_lock;static struct kmem_cache *mqueue_inode_cachep; -static struct vfsmount *mqueue_mnt; - -static unsigned int queues_count; -static unsigned int queues_max =DFLT_QUEUESMAX ; -static unsigned int msg_max =DFLT_MSGMAX ; -static unsigned int msgsize_max =DFLT_MSGSIZEMAX ;static struct ctl_table_header * mq_sysctl_table;
ipc/mqueue.c static inline struct mqueue_inode_info *MQUEUE_I(struct inode *inode) · line 107
Quoted 10 lines
ipc/mqueue.c +7
-0@@ -109,11 +98,25 @@ static inline struct mqueue_inode_info *MQUEUE_I(struct inode *inode) return container_of(inode, struct mqueue_inode_info, vfs_inode); } +void mq_init_ns (struct ipc_namespace *ns) { + ns->mq_queues_count =0 ; + ns->mq_queues_max =DFLT_QUEUESMAX ; + ns->mq_msg_max =DFLT_MSGMAX ; + ns->mq_msgsize_max =DFLT_MSGSIZEMAX ; + ns->mq_mnt =mntget (init_ipc_ns.mq_mnt); +}
Heh, I read that as a structure definition at first! Could you put that bracket on a new line.
This part of the patch is nice. The -'s are next to the +'s and it is easy to audit.
The rest of the patch looks good. Mostly simple substitutions, and it is pretty obvious what is going on.
-- Dave
Re: [PATCH 1/2] mqueue ns: move mqueue_mnt into struct ipc_namespace
Quoted 1 line · Dave Hansen
On Wed, 2008-12-17 at 11:55 -0600, Serge E. Hallyn wrote:
Thanks for taking look.
...
Quoted 5 lines
Quoted 5 lines
include/linux/ipc_namespace.h +4
-1-#if defined(CONFIG_SYSVIPC ) && defined(CONFIG_IPC_NS ) +#ifdef CONFIG_POSIX_MQUEUE +extern void mq_init_ns (struct ipc_namespace *ns); +/* default values */ +#define DFLT_QUEUESMAX 256/* max number of message queues */
Quoted 8 lines
include/linux/ipc_namespace.h struct ipc_namespace { · line 86
Quoted 6 lines
include/linux/ipc_namespace.h +6
-0+#define DFLT_MSGMAX 10/* max number of messages in each queue */ +#define HARD_MSGMAX (131072/sizeof(void *)) +#define DFLT_MSGSIZEMAX 8192/* max message size */ +#else +#define mq_init_ns (ns) +#endif Do all these empty suckers need do{}while(0)'s?
Couldn't hurt for future-proofing. Will do, thanks.
Quoted 4 lines
Quoted 4 lines
ipc/mqueue.c +0
-0@@ -109,11 +98,25 @@ static inline struct mqueue_inode_info *MQUEUE_I(struct inode *inode) return container_of(inode, struct mqueue_inode_info, vfs_inode); }
Quoted 10 lines
ipc/mqueue.c static inline struct mqueue_inode_info *MQUEUE_I(struct inode *inode) · line 107
Quoted 7 lines
ipc/mqueue.c +7
-0+void mq_init_ns (struct ipc_namespace *ns) { + ns->mq_queues_count =0 ; + ns->mq_queues_max =DFLT_QUEUESMAX ; + ns->mq_msg_max =DFLT_MSGMAX ; + ns->mq_msgsize_max =DFLT_MSGSIZEMAX ; + ns->mq_mnt =mntget (init_ipc_ns.mq_mnt); +}Heh, I read that as a structure definition at first! Could you put that bracket on a new line.
That's actually taken care of in the followup patch - same with mq_exit_ns. Sorry about that. (Still I'd rather not bother with the patch flux to change it just in this patch, since whenever I do that I inevitably mess up something else)
Quoted 5 lines
This part of the patch is nice. The -'s are next to the +'s and it is easy to audit.
The rest of the patch looks good. Mostly simple substitutions, and it is pretty obvious what is going on.
Thanks. I'm hoping that Pavel and Eric will give their input and maybe we can put this thing to bed :)
-serge
Re: [PATCH 1/2] mqueue ns: move mqueue_mnt into struct ipc_namespace
Quoted 9 lines · Serge E. Hallyn
Quoted 9 lines
ipc/mqueue.c static inline struct mqueue_inode_info *MQUEUE_I(struct inode *inode) · line 107
Quoted 7 lines
ipc/mqueue.c +7
-0+void mq_init_ns (struct ipc_namespace *ns) { + ns->mq_queues_count =0 ; + ns->mq_queues_max =DFLT_QUEUESMAX ; + ns->mq_msg_max =DFLT_MSGMAX ; + ns->mq_msgsize_max =DFLT_MSGSIZEMAX ; + ns->mq_mnt =mntget (init_ipc_ns.mq_mnt); +}Heh, I read that as a structure definition at first! Could you put
Quoted 7 lines
that
Quoted 1 line
bracket on a new line.
That's actually taken care of in the followup patch - same with
mq_exit_ns. Sorry about that. (Still I'd rather not bother with the patch flux to change it just in this patch, since whenever I do that I inevitably mess up something else)
Ahh, ok. I guess it is OK since it is temporary.
-- Dave
PATCH 2/2 ipc namespaces: implement support for posix msqueues
In reply to Serge E. HallynImplement multiple mounts of the mqueue file system, and link it to usage of CLONE_NEWIPC.
Each ipc ns has a corresponding mqueuefs superblock. When a user does clone(CLONE_NEWIPC) or unshare(CLONE_NEWIPC), the unshare will cause an internal mount of a new mqueuefs sb linked to the new ipc ns.
When a user does 'mount -t mqueue mqueue /dev/mqueue', he mounts the mqueuefs superblock.
Posix message queues can be worked with both through the mq_* system calls (see mq_overview(7)), and through the VFS through the mqueue mount. Any usage of mq_open() and friends will work with the acting task's ipc namespace. Any actions through the VFS will work with the mqueuefs in which the file was created. So if a user doesn't remount mqueuefs after unshare(CLONE_NEWIPC), mq_open("/ab") will not be reflected in "ls /dev/mqueue".
If task a mounts mqueue for ipc_ns:1, then clones task b with a new ipcns, ipcns:2, and then task a is the last task in ipc_ns:1 to exit, then (1) ipc_ns:1 will be freed, (2) it's superblock will live on until task b umounts the corresponding mqueuefs, and vfs actions will continue to succeed, but (3) sb->s_fs_info will be NULL for the sb corresponding to the deceased ipc_ns:1.
Changelog:
Dec 17: removed unused static fn (get_ipcns_from_sb)
--- include/linux/ipc_namespace.h | 16 ++--- ipc/mqueue.c | 140 ++++++++++++++++++++++++++++++++--------- ipc/msgutil.c | 8 +-- ipc/namespace.c | 25 ++++++-- ipc/util.h | 6 +- 5 files changed, 144 insertions(+), 51 deletions(-)
include/linux/ipc_namespace.h +7 -9
@@ -25,7 +25,7 @@ struct ipc_ids { };struct ipc_namespace { -struct kref kref; +atomic_t count;struct ipc_ids ids[3 ];int sem_ctls[4 ];
@@ -56,6 +56,7 @@ struct ipc_namespace {extern struct ipc_namespace init_ipc_ns;extern atomic_t nr_ipc_ns; +extern spinlock_t mq_lock;#if defined(CONFIG_POSIX_MQUEUE ) || defined(CONFIG_SYSVIPC )#define INIT_IPC_NS (ns) .ns = &init_ipc_ns,#else
@@ -75,18 +76,18 @@ extern int ipcns_notify(unsigned long);#endif /* CONFIG_SYSVIPC */ #ifdef CONFIG_POSIX_MQUEUE -extern void mq_init_ns (struct ipc_namespace *ns); +extern int mq_init_ns (struct ipc_namespace *ns);/* default values */ #define DFLT_QUEUESMAX 256/* max number of message queues */ #define DFLT_MSGMAX 10/* max number of messages in each queue */ #define HARD_MSGMAX (131072/sizeof(void *))#define DFLT_MSGSIZEMAX 8192/* max message size */ #else -#define mq_init_ns (ns) +#define mq_init_ns (ns) (0)#endif #if defined(CONFIG_IPC_NS ) -extern void free_ipc_ns (struct kref *kref); +extern void free_ipc_ns (struct ipc_namespace *ns);extern struct ipc_namespace *copy_ipcs (unsigned long flags,struct ipc_namespace *ns); extern void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,
@@ -96,14 +97,11 @@ extern void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,static inline struct ipc_namespace *get_ipc_ns (struct ipc_namespace *ns) {if (ns) -kref_get (&ns->kref); +atomic_inc (&ns->count);return ns; } -static inline void put_ipc_ns (struct ipc_namespace *ns) -{ -kref_put (&ns->kref, free_ipc_ns); -} +extern void put_ipc_ns (struct ipc_namespace *ns);#else static inline struct ipc_namespace *copy_ipcs (unsigned long flags,struct ipc_namespace *ns)
ipc/mqueue.c +111 -29
258 lines not on this page · raw message
ipc/msgutil.c +3 -5
@@ -18,18 +18,16 @@#include "util.h" +spinlock_t mq_lock; +/* * The next 2 defines are here bc this is the only file * compiled when either CONFIG_SYSVIPC and CONFIG_POSIX_MQUEUE * and not CONFIG_IPC_NS. */ struct ipc_namespace init_ipc_ns = { - .kref = { - .refcount =ATOMIC_INIT (2 ), - }, + .count =ATOMIC_INIT (2 ), #ifdef CONFIG_POSIX_MQUEUE - .mq_mnt =NULL , - .mq_queues_count =0 , .mq_queues_max =DFLT_QUEUESMAX , .mq_msg_max =DFLT_MSGMAX , .mq_msgsize_max = DFLT_MSGSIZEMAX,
ipc/namespace.c +19 -6
@@ -9,23 +9,31 @@#include <linux/rcupdate.h> #include <linux/nsproxy.h> #include <linux/slab.h> +#include <linux/fs.h> +#include <linux/mount.h> #include "util.h" static struct ipc_namespace *clone_ipc_ns (struct ipc_namespace *old_ns) {struct ipc_namespace *ns; +int err; ns =kmalloc (sizeof (struct ipc_namespace ),GFP_KERNEL );if (ns ==NULL )return ERR_PTR (-ENOMEM ); +atomic_set (&ns->count,1 ); + err =mq_init_ns (ns); +if (err) { +kfree (ns); +return ERR_PTR (err); + }atomic_inc (&nr_ipc_ns);sem_init_ns (ns);msg_init_ns (ns);shm_init_ns (ns); -mq_init_ns (ns);/* * msgmni has already been computed for the new ipc ns.
@@ -35,7 +43,6 @@ static struct ipc_namespace *clone_ipc_ns(struct ipc_namespace *old_ns)ipcns_notify (IPCNS_CREATED );register_ipcns_notifier (ns); -kref_init (&ns->kref);return ns; }
@@ -85,11 +92,18 @@ void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,up_write (&ids->rw_mutex); } -void free_ipc_ns (struct kref *kref) +void put_ipc_ns (struct ipc_namespace *ns) { -struct ipc_namespace *ns; +if (ns &&atomic_dec_and_lock (&ns->count, &mq_lock)) { +mq_clear_sbinfo (ns); +spin_unlock (&mq_lock); +mq_put_mnt (ns); +free_ipc_ns (ns); + } +} - ns = container_of(kref, struct ipc_namespace, kref); +void free_ipc_ns (struct ipc_namespace *ns) +{/* * Unregistering the hotplug notifier at the beginning guarantees * that the ipc namespace won't be freed while we are inside the
@@ -102,7 +116,6 @@ void free_ipc_ns(struct kref *kref)sem_exit_ns (ns);msg_exit_ns (ns);shm_exit_ns (ns); -mq_exit_ns (ns);kfree (ns);atomic_dec (&nr_ipc_ns);
ipc/util.h +4 -2
@@ -21,9 +21,11 @@ void shm_init (void);struct ipc_namespace ;#ifdef CONFIG_POSIX_MQUEUE -void mq_exit_ns (struct ipc_namespace *ns); +extern void mq_clear_sbinfo (struct ipc_namespace *ns); +extern void mq_put_mnt (struct ipc_namespace *ns);#else -#define mq_exit_ns (ns) +#define mq_clear_sbinfo (ns) +#define mq_put_mnt (ns)#endif #ifdef CONFIG_SYSVIPC
Signature · 1 line
1.5.4.3
Re: [PATCH 2/2] ipc namespaces: implement support for posix msqueues
ipc/namespace.c void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids, · line 103
Quoted 11 lines · Serge E. Hallyn
ipc/namespace.c +8
-2-void free_ipc_ns (struct kref *kref) +void put_ipc_ns (struct ipc_namespace *ns) { -struct ipc_namespace *ns; +if (ns &&atomic_dec_and_lock (&ns->count, &mq_lock)) { +mq_clear_sbinfo (ns); +spin_unlock (&mq_lock); +mq_put_mnt (ns); +free_ipc_ns (ns); + } +}
OK, I see now why you wanted to remove the kref and add an atomic_t. That's probably worth a sentence in the patch description.
I know that atomic_dec_and_lock() can be a nice logical thing to do, but is it necessary here? Basically, is there something that happens under mq_lock that can re-raise the ns->count?
-- Dave
Re: [PATCH 2/2] ipc namespaces: implement support for posix msqueues
Quoted 15 lines · Dave Hansen
ipc/namespace.c void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids, · line 103
Quoted 11 lines · Serge E. Hallyn
ipc/namespace.c +8
-2-void free_ipc_ns (struct kref *kref) +void put_ipc_ns (struct ipc_namespace *ns) { -struct ipc_namespace *ns; +if (ns &&atomic_dec_and_lock (&ns->count, &mq_lock)) { +mq_clear_sbinfo (ns); +spin_unlock (&mq_lock); +mq_put_mnt (ns); +free_ipc_ns (ns); + } +}OK, I see now why you wanted to remove the kref and add an
atomic_t. That's probably worth a sentence in the patch description.
Good point, will do.
Quoted 3 lines
I know that
atomic_dec_and_lock()can be a nice logical thing to do, but is it necessary here? Basically, is there something that happens undermq_lockthat can re-raise thens->count?
A racing task from another ipc namespace can open a mq file through the VFS and thereby bump the ipc's refcount. Will comment that here and in the patch description.
thanks,
-serge
Re: [PATCH 2/2] ipc namespaces: implement support for posix msqueues
In reply to Serge E. HallynJust a couple of nits.
Serge E. Hallyn [[email protected]] wrote:
| Implement multiple mounts of the mqueue file system, and
| link it to usage of CLONE_NEWIPC.
|
| Each ipc ns has a corresponding mqueuefs superblock. When
| a user does clone(CLONE_NEWIPC) or unshare(CLONE_NEWIPC), the
| unshare will cause an internal mount of a new mqueuefs sb
| linked to the new ipc ns.
|
| When a user does 'mount -t mqueue mqueue /dev/mqueue', he
| mounts the mqueuefs superblock.
|
| Posix message queues can be worked with both through the
| mq_* system calls (see mq_overview(7)), and through the VFS
| through the mqueue mount. Any usage of mq_open() and friends
| will work with the acting task's ipc namespace. Any actions
| through the VFS will work with the mqueuefs in which the
| file was created. So if a user doesn't remount mqueuefs
| after unshare(CLONE_NEWIPC), mq_open("/ab") will not be
| reflected in "ls /dev/mqueue".
|
| If task a mounts mqueue for ipc_ns:1, then clones task b with
| a new ipcns, ipcns:2, and then task a is the last task in
| ipc_ns:1 to exit, then (1) ipc_ns:1 will be freed, (2) it's
| superblock will live on until task b umounts the corresponding
| mqueuefs, and vfs actions will continue to succeed, but (3)
| sb->s_fs_info will be NULL for the sb corresponding to the
| deceased ipc_ns:1.
|
| Changelog:
| Dec 17: removed unused static fn (get_ipcns_from_sb)
|
| Signed-off-by: Cedric Le Goater <[email protected]>
| Signed-off-by: Serge E. Hallyn <[email protected]>
| ---
| include/linux/ipc_namespace.h | 16 ++---
| ipc/mqueue.c | 140 ++++++++++++++++++++++++++++++++---------
| ipc/msgutil.c | 8 +--
| ipc/namespace.c | 25 ++++++--
| ipc/util.h | 6 +-
| 5 files changed, 144 insertions(+), 51 deletions(-)
|
| diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h
| index 532598f..74f1ae2 100644
| --- a/include/linux/ipc_namespace.h
| +++ b/include/linux/ipc_namespace.h
| @@ -25,7 +25,7 @@ struct ipc_ids {
| };
|
| struct ipc_namespace {
| - struct kref kref;
| + atomic_t count;
| struct ipc_ids ids[3];
|
| int sem_ctls[4];
| @@ -56,6 +56,7 @@ struct ipc_namespace {
| extern struct ipc_namespace init_ipc_ns;
| extern atomic_t nr_ipc_ns;
|
| +extern spinlock_t mq_lock;
| #if defined(CONFIG_POSIX_MQUEUE) || defined(CONFIG_SYSVIPC)
| #define INIT_IPC_NS(ns) .ns = &init_ipc_ns,
| #else
| @@ -75,18 +76,18 @@ extern int ipcns_notify(unsigned long);
| #endif /* CONFIG_SYSVIPC */
|
| #ifdef CONFIG_POSIX_MQUEUE
| -extern void mq_init_ns(struct ipc_namespace *ns);
| +extern int mq_init_ns(struct ipc_namespace *ns);
| /* default values */
| #define DFLT_QUEUESMAX 256 /* max number of message queues */
| #define DFLT_MSGMAX 10 /* max number of messages in each queue */
| #define HARD_MSGMAX (131072/sizeof(void *))
| #define DFLT_MSGSIZEMAX 8192 /* max message size */
| #else
| -#define mq_init_ns(ns)
| +#define mq_init_ns(ns) (0)
| #endif
|
| #if defined(CONFIG_IPC_NS)
| -extern void free_ipc_ns(struct kref *kref);
| +extern void free_ipc_ns(struct ipc_namespace *ns);
| extern struct ipc_namespace *copy_ipcs(unsigned long flags,
| struct ipc_namespace *ns);
| extern void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,
| @@ -96,14 +97,11 @@ extern void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,
| static inline struct ipc_namespace *get_ipc_ns(struct ipc_namespace *ns)
| {
| if (ns)
| - kref_get(&ns->kref);
| + atomic_inc(&ns->count);
| return ns;
| }
|
| -static inline void put_ipc_ns(struct ipc_namespace *ns)
| -{
| - kref_put(&ns->kref, free_ipc_ns);
| -}
| +extern void put_ipc_ns(struct ipc_namespace *ns);
| #else
| static inline struct ipc_namespace *copy_ipcs(unsigned long flags,
| struct ipc_namespace *ns)
| diff --git a/ipc/mqueue.c b/ipc/mqueue.c
| index 01d64a0..6b235c1 100644
| --- a/ipc/mqueue.c
| +++ b/ipc/mqueue.c
| @@ -88,7 +88,6 @@ static const struct file_operations mqueue_file_operations;
| static struct super_operations mqueue_super_ops;
| static void remove_notification(struct mqueue_inode_info *info);
|
| -static spinlock_t mq_lock;
| static struct kmem_cache *mqueue_inode_cachep;
|
| static struct ctl_table_header * mq_sysctl_table;
| @@ -98,25 +97,30 @@ static inline struct mqueue_inode_info *MQUEUE_I(struct inode *inode)
| return container_of(inode, struct mqueue_inode_info, vfs_inode);
| }
|
| -void mq_init_ns(struct ipc_namespace *ns) {
| - ns->mq_queues_count = 0;
| - ns->mq_queues_max = DFLT_QUEUESMAX;
| - ns->mq_msg_max = DFLT_MSGMAX;
| - ns->mq_msgsize_max = DFLT_MSGSIZEMAX;
| - ns->mq_mnt = mntget(init_ipc_ns.mq_mnt);
| +/*
| + * This routine should be called with the mq_lock held.
| + */
| +static inline struct ipc_namespace *__get_ns_from_ino(struct inode *inode)Nit. Lot of places in kernel use 'ino' to represent the numeric inode number. Maybe better to name get_ns_from_inode().
| +{
| + return get_ipc_ns(inode->i_sb->s_fs_info);
| }
|
| -void mq_exit_ns(struct ipc_namespace *ns) {
| - /* will need to clear out ns->mq_mnt->mnt_sb->s_fs_info here */
| - mntput(ns->mq_mnt);
| +static inline struct ipc_namespace *get_ns_from_ino(struct inode *inode)| +{
| + struct ipc_namespace *ns;
| +
| + spin_lock(&mq_lock);
| + ns = __get_ns_from_ino(inode);
| + spin_unlock(&mq_lock);
| + return ns;
| }
|
| -static struct inode *mqueue_get_inode(struct super_block *sb, int mode,
| - struct mq_attr *attr)
| +static struct inode *mqueue_get_inode(struct super_block *sb,
| + struct ipc_namespace *ipc_ns, int mode,
| + struct mq_attr *attr)
| {
| struct user_struct *u = current_user();
| struct inode *inode;
| - struct ipc_namespace *ipc_ns = &init_ipc_ns;
|
| inode = new_inode(sb);
| if (inode) {
| @@ -192,30 +196,76 @@ out_inode:
| static int mqueue_fill_super(struct super_block *sb, void *data, int silent)
| {
| struct inode *inode;
| + struct ipc_namespace *ns = data;
| + int error = 0;
|
| sb->s_blocksize = PAGE_CACHE_SIZE;
| sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
| sb->s_magic = MQUEUE_MAGIC;
| sb->s_op = &mqueue_super_ops;
|
| - inode = mqueue_get_inode(sb, S_IFDIR | S_ISVTX | S_IRWXUGO, NULL);
| - if (!inode)
| - return -ENOMEM;
| + inode = mqueue_get_inode(sb, ns, S_IFDIR | S_ISVTX | S_IRWXUGO,
| + NULL);
| + if (!inode) {
| + error = -ENOMEM;
| + goto out;
| + }
|
| sb->s_root = d_alloc_root(inode);
| if (!sb->s_root) {
| iput(inode);
| - return -ENOMEM;
| + error = -ENOMEM;
| }
|
| - return 0;
| +out:
| + return error;
| +}
| +
| +static int compare_sb_single_ns(struct super_block *sb, void *data)
| +{
| + return sb->s_fs_info == data;
| +}
| +
| +static int set_sb_single_ns(struct super_block *sb, void *data)
| +{
| + sb->s_fs_info = data;
| + return set_anon_super(sb, NULL);
| +}
| +
| +static int get_sb_single_ns(struct file_system_type *fs_type,
| + int flags, void *data,
| + int (*fill_super)(struct super_block *, void *, int),
| + struct vfsmount *mnt)
| +{
| + struct super_block *s;
| + int error;
| +
| + s = sget(fs_type, compare_sb_single_ns, set_sb_single_ns, data);
| + if (IS_ERR(s))
| + return PTR_ERR(s);
| + if (!s->s_root) {
| + s->s_flags = flags;
| + error = fill_super(s, data, flags & MS_SILENT ? 1 : 0);
| + if (error) {
| + up_write(&s->s_umount);
| + deactivate_super(s);
| + return error;
| + }
| + s->s_flags |= MS_ACTIVE;
| + }
| + do_remount_sb(s, flags, data, 0);
| + return simple_set_mnt(mnt, s);
| }
|
| static int mqueue_get_sb(struct file_system_type *fs_type,
| int flags, const char *dev_name,
| void *data, struct vfsmount *mnt)
| {
| - return get_sb_single(fs_type, flags, data, mqueue_fill_super, mnt);
| + if (flags & MS_KERNMOUNT)
| + return get_sb_single_ns(fs_type, flags, data,
| + mqueue_fill_super, mnt);
| + return get_sb_single_ns(fs_type, flags, current->nsproxy->ipc_ns,
| + mqueue_fill_super, mnt);nit. This maybe easier to read with an ns local variable and a single call to get_sb_single_ns().
Suka
Re: [PATCH 2/2] ipc namespaces: implement support for posix msqueues
Quoted 11 lines · Sukadev Bhattiprolu
| static int mqueue_get_sb(struct file_system_type *fs_type,
| int flags, const char *dev_name,
| void *data, struct vfsmount *mnt)
| {
| - return get_sb_single(fs_type, flags, data, mqueue_fill_super, mnt);
| + if (flags & MS_KERNMOUNT)
| + return get_sb_single_ns(fs_type, flags, data,
| + mqueue_fill_super, mnt);
| + return get_sb_single_ns(fs_type, flags, current->nsproxy->ipc_ns,
| + mqueue_fill_super, mnt);Quoted 2 lines
nit. This maybe easier to read with an ns local variable and a single call to
get_sb_single_ns().
The first time I read this I completely misunderstood what you meant. Yes, that's nicer. Thanks for taking a look.
New version coming tomorrow (I want to stress this a bit).
thanks,
-serge
PATCH 1/4 posix message queue namespaces: first test
In reply to Serge E. Hallyn(Subrata, I'm sending these testcases now so people can use them, but they should not yet be considered for integration into ltp of course, and when they are, the kernel version test will need to be switched to at least 2.6.29)
Add the first test for posix message queue namespaces, plus a test to detect whether they are enabled.
Note that the kernel version check is currently bogus - these are not yet enabled in 2.6.28. But a fella's gotta test.
Based on older version by Nadia Derbey.
--- testcases/kernel/containers/Makefile | 2 +- testcases/kernel/containers/README | 4 + testcases/kernel/containers/container_test.sh | 9 ++ testcases/kernel/containers/mqns/Makefile | 41 +++++++ .../kernel/containers/mqns/check_mqns_enabled.c | 65 ++++++++++ testcases/kernel/containers/mqns/mqns.h | 13 ++ testcases/kernel/containers/mqns/mqns_01.c | 123 ++++++++++++++++++++ testcases/kernel/containers/mqns/runmqnstest.sh | 39 ++++++ 8 files changed, 295 insertions(+), 1 deletions(-) create mode 100644 testcases/kernel/containers/mqns/Makefile create mode 100644 testcases/kernel/containers/mqns/check_mqns_enabled.c create mode 100644 testcases/kernel/containers/mqns/mqns.h create mode 100644 testcases/kernel/containers/mqns/mqns_01.c create mode 100644 testcases/kernel/containers/mqns/runmqnstest.sh
testcases/kernel/containers/Makefile +1 -1
@@ -18,7 +18,7 @@ ## ## ################################################################################ -SUBDIRS := libclone utsname sysvipc pidns netns +SUBDIRS := libclone utsname sysvipc pidns netns mqns all: check_for_unshare @set -e; for i in $(SUBDIRS); do $(MAKE) -C $$i $@; done
testcases/kernel/containers/README +4 -0
@@ -37,6 +37,10 @@ each functionality README file for detail: sysvipc/* Contains all the testcases related to IPC NS tests. +posixmq/* + Contains all the testcases related to POSIX MQ NS tests. These + are strictly speaking a part of the ipc namespaces, but can be + enabled in the kernel without SYSV IPC support. utsname/* Contains all the testcases related to utsname tests. libclone/*
testcases/kernel/containers/container_test.sh +9 -0
@@ -54,6 +54,15 @@ else echo "Process id namespaces not enabled in kernel. Not running pidns tests." fi +check_mqns_enabled +if [ $? -eq 0 ]; then + echo "Running POSIX message queue tests." + runmqnstest.sh +else + echo "Posix message queues or ipc namespaces not enabled in kernel." + echo "Not running mqns tests." +fi + check_netns_enabled if [ $? -eq 0 ]; then echo "Running netns tests."
testcases/kernel/containers/mqns/Makefile new file +41 -0
41 lines not on this page · raw message
testcases/kernel/containers/mqns/check_mqns_enabled.c new file +65 -0
65 lines not on this page · raw message
testcases/kernel/containers/mqns/mqns.h new file +13 -0
@@ -0,0 +1,13 @@ +#ifndef __MQNS_H +#define __MQNS_H + +#include <sys/mount.h> +#include <mqueue.h> +#include <test.h> +#include <libclone.h> + + +#define DEV_MQUEUE "/dev/mqueue" +#define SLASH_MQ1 "/MQ1" + +#endif /* __MQNS_H */
testcases/kernel/containers/mqns/mqns_01.c new file +123 -0
123 lines not on this page · raw message
testcases/kernel/containers/mqns/runmqnstest.sh new file +39 -0
@@ -0,0 +1,39 @@ +#!/bin/sh +################################################################################ +## ## +## Copyright (c) International Business Machines Corp., 2007 ## +## ## +## This program is free software; you can redistribute it and#or modify ## +## it under the terms of the GNU General Public License as published by ## +## the Free Software Foundation; either version 2 of the License, or ## +## (at your option) any later version. ## +## ## +## This program is distributed in the hope that it will be useful, but ## +## WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ## +## or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ## +## for more details. ## +## ## +## You should have received a copy of the GNU General Public License ## +## along with this program; if not, write to the Free Software ## +## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ## +## ## +################################################################################ + +exit_code=0 +tests_list='mqns_01' + +for t in $tests_list +do + $t + if [ $? -ne 0 ]; then + exit_code="$?" + exit $exit_code + fi + $t -clone + if [ $? -ne 0 ]; then + exit_code="$?" + exit $exit_code + fi +done + +exit $exit_code
Signature · 1 line
1.6.0.5
PATCH 2/4 posix mqns: test parent to child mq access
In reply to Serge E. HallynIt's kind of redundant with test 01 since there is no hierarchical relationship between ipc namespaces - they are all completely isolated. But heck it can't hurt.
--- testcases/kernel/containers/mqns/mqns_02.c | 138 +++++++++++++++++++++++ testcases/kernel/containers/mqns/runmqnstest.sh | 2 +- 2 files changed, 139 insertions(+), 1 deletions(-) create mode 100644 testcases/kernel/containers/mqns/mqns_02.c mode change 100644 => 100755 testcases/kernel/containers/mqns/runmqnstest.sh
testcases/kernel/containers/mqns/mqns_02.c new file +138 -0
@@ -0,0 +1,138 @@ +/* +* Copyright (c) International Business Machines Corp., 2007 +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +* the GNU General Public License for more details. +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +* +* Author: Nadia Derbey <[email protected]> +* +* Check mqns isolation: child mqns cannot be accessed from father +* +* Mount mqueue fs +* unshare +* In unshared process: +* Mount newinstance mqueuefs +* Create a posix mq -->mq1 +* Check that mq1 is not readable from father +* +* Changelog: +* Dec 16: accomodate new mqns semantics (Serge Hallyn) + +***************************************************************************/ + +#define _GNU_SOURCE 1 +#include <sys/wait.h> +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> +#include "mqns.h" + +char *TCID ="posixmq_namespace_02" ; +int TST_TOTAL =1 ; + +int p1[2 ]; +int p2[2 ]; + +int check_mqueue (void *vtest) +{ +char buf[3 ]; +mqd_t mqd; + +close (p1[1 ]); +close (p2[0 ]); + +read (p1[0 ], buf,3 ); + + mqd =mq_open (SLASH_MQ1 ,O_RDWR |O_CREAT |O_EXCL ,0777 ,NULL ); +if (mqd ==-1 ) { +write (p2[1 ],"mqfail" ,7 ); +tst_exit (3 ); + } + +write (p2[1 ],"mqopen" ,7 ); + +read (p1[0 ], buf,5 ); + +/* destroy the mqueue */ +mq_close (mqd); +mq_unlink (SLASH_MQ1 ); + +write (p2[1 ],"done" ,5 ); + +tst_exit (0 ); + +/* NOT REACHED */ +return 0 ; +} + + +int main (int argc,char *argv[]) +{ +int r; +mqd_t mqd; +char buf[7 ]; +int use_clone =T_UNSHARE ; + +if (argc ==2 &&strcmp (argv[1 ],"-clone" ) ==0 ) { +tst_resm (TINFO ,"Testing posix mq namespaces through clone(2).\n" ); + use_clone =T_CLONE ; + }else +tst_resm (TINFO ,"Testing posix mq namespaces through unshare(2).\n" ); + +if (pipe (p1) ==-1 ) {perror ("pipe" );exit (EXIT_FAILURE ); } +if (pipe (p2) ==-1 ) {perror ("pipe" );exit (EXIT_FAILURE ); } + +/* fire off the test */ + r =do_clone_unshare_test (use_clone,CLONE_NEWIPC , check_mqueue,NULL ); +if (r <0 ) { +tst_resm (TFAIL ,"failed clone/unshare\n" ); +tst_exit (1 ); + } + +tst_resm (TINFO ,"Checking namespaces isolation (child to parent)\n" ); + +close (p1[0 ]); +close (p2[1 ]); +write (p1[1 ],"go" ,3 ); + +read (p2[0 ], buf,7 ); +if (!strcmp (buf,"mqfail" )) { +tst_resm (TFAIL ,"child process could not create mqueue\n" ); +umount (DEV_MQUEUE ); +tst_exit (TFAIL ); + }else if (strcmp (buf,"mqopen" )) { +tst_resm (TFAIL ,"child process could not create mqueue\n" ); +umount (DEV_MQUEUE ); +tst_exit (TFAIL ); + } + + mqd =mq_open (SLASH_MQ1 ,O_RDONLY ); +if (mqd ==-1 ) { + r =TPASS ; +tst_resm (TPASS ,"Father process doesn't see mqueue\n" ); + }else { + r =TFAIL ; +tst_resm (TFAIL ,"Father process found mqueue\n" ); +mq_close (mqd); + } + +write (p1[1 ],"cont" ,5 ); +read (p2[0 ], buf,7 ); + +tst_exit (r); + +/* NOT REACHED */ +return 0 ; +}
testcases/kernel/containers/mqns/runmqnstest.sh +1 -1
@@ -20,7 +20,7 @@ ################################################################################ exit_code=0 -tests_list='mqns_01' +tests_list='mqns_01 mqns_02' for t in $tests_list do
Signature · 1 line
1.6.0.5
PATCH 3/4 posix mqns: test vfs and mq interaction
In reply to Serge E. HallynTest that the interaction between mqueuefs and ipc namespaces is correct (one mqueuefs for each ipc namespace, living at least as long as the namespace)
--- testcases/kernel/containers/mqns/mqns.h | 1 + testcases/kernel/containers/mqns/mqns_03.c | 196 +++++++++++++++++++++++ testcases/kernel/containers/mqns/runmqnstest.sh | 2 +- 3 files changed, 198 insertions(+), 1 deletions(-) create mode 100644 testcases/kernel/containers/mqns/mqns_03.c
testcases/kernel/containers/mqns/mqns.h +1 -0
@@ -9,5 +9,6 @@#define DEV_MQUEUE "/dev/mqueue"#define SLASH_MQ1 "/MQ1" +#define SLASH_MQ2 "/MQ2"#endif /* __MQNS_H */
testcases/kernel/containers/mqns/mqns_03.c new file +196 -0
196 lines not on this page · raw message
testcases/kernel/containers/mqns/runmqnstest.sh +1 -1
@@ -20,7 +20,7 @@ ################################################################################ exit_code=0 -tests_list='mqns_01 mqns_02' +tests_list='mqns_01 mqns_02 mqns_03' for t in $tests_list do
Signature · 1 line
1.6.0.5
PATCH 4/4 posix mqns: test that user mount of posixmq survivies the ipcns
In reply to Serge E. HallynIf the mqueuefs for a ipc namespace is mounted in a mounts namespace which outlives the ipcns, then the view of the fs remains valid until someone umounts it. Run some tests to make sure that all is sane.
--- testcases/kernel/containers/mqns/mqns.h | 1 + testcases/kernel/containers/mqns/mqns_04.c | 169 +++++++++++++++++++++++ testcases/kernel/containers/mqns/runmqnstest.sh | 2 +- 3 files changed, 171 insertions(+), 1 deletions(-) create mode 100644 testcases/kernel/containers/mqns/mqns_04.c
testcases/kernel/containers/mqns/mqns.h +1 -0
@@ -8,6 +8,7 @@#define DEV_MQUEUE "/dev/mqueue" +#define DEV_MQUEUE2 "/dev/mqueue2"#define SLASH_MQ1 "/MQ1"#define SLASH_MQ2 "/MQ2"
testcases/kernel/containers/mqns/mqns_04.c new file +169 -0
169 lines not on this page · raw message
testcases/kernel/containers/mqns/runmqnstest.sh +1 -1
@@ -20,7 +20,7 @@ ################################################################################ exit_code=0 -tests_list='mqns_01 mqns_02 mqns_03' +tests_list='mqns_01 mqns_02 mqns_03 mqns_04' for t in $tests_list do
Signature · 1 line
1.6.0.5