Skip to contentSkip to conversation

Virtualization/containers: introduction

PATCH In review Kirill Korotaev 15 participants 80 messages Feb 6, 2006 – Feb 20, 2006

Subsystems (19)
Networking Arm (32-bit) VFS USB +15 more subsystems
lore
Share & tools
lore
#0 lore raw

PATCH 1/4 Virtualization/containers: introduction

Hello,

I tried to take into account all the comments from you guys (thanks a lot for them!) and prepared a new version of virtualization patches. I will send only 4 patches today, just not to overflow everyone and keep it clear/tidy/possible to review.

This patch introduces some abstract container kernel structure and a number of operations on it.

The important properties of the proposed container implementation:
- each container has unique ID in the system
- each process in the kernel can belong to one container only
- effective container pointer (econtainer()) is used on the task to avoid insertion of additional argument "container" to all functions where it is required.
- kernel compilation with disabled virtualization should result in old good linux kernel

Patches following this one will be used for virtualization of the kernel resources based on this container infrastructure, including those VPID patches I sent before. Every virtualized resource can be given separate config option if needed (just give me to know if it is desired).

Kirill

P.S. I understand that this virtualization spam can be unintersting for some of you, just give me to know if you want to be removed from CC.

P.P.S. All patches are against 2.6.16-rc2-git2 and compile when virtualization=n.

include/linux/container.h renamed from include/linux/container.h.vpsinfo +59 -0
@@ -0,0 +1,59 @@
+#ifndef __LINUX_CONTAINER_H__
+#define __LINUX_CONTAINER_H__
+
+#include <linux/config.h>
+#include <asm/types.h>
+#include <asm/atomic.h>
+
+struct task_struct;
+
+struct container {
+	u32 id;
+	struct task_struct *init_task;
+	atomic_t refcnt;
+};
+
+extern struct container init_container;
+
+#ifdef CONFIG_CONTAINER
+
+static inline struct container *get_container(struct container *cont)
+{
+	atomic_inc(&cont->refcnt);
+	return cont;
+}
+
+static inline void put_container(struct container *cont)
+{
+	atomic_dec(&container->refcnt);
+}
+
+#include <linux/sched.h>
+#include <asm/current.h>
+
+static inline struct container *set_econtainer(struct container *cont)
+{
+	struct container *ret;
+
+	ret = current->econtainer;
+	current->econtainer = cont;
+	return ret;
+}
+
+#define econtainer()		(current->econtainer)
+
+extern void init_containers(void);
+
+#else	/* CONFIG_CONTAINER */
+
+#define get_container(cont)	(NULL)
+#define put_container(cont)	do { } while (0)
+
+#define set_econtainer(cont)	((struct container *)NULL)
+#define econtainer()		(NULL)
+
+#define init_containers()	do { } while (0)
+
+#endif	/* CONFIG_CONTAINER */
+
+#endif
include/linux/init_task.h renamed from include/linux/init_task.h.vpsinfo +3 -0
@@ -3,6 +3,7 @@
 
 #include <linux/file.h>
 #include <linux/rcupdate.h>
+#include <linux/container.h>
 
 #define INIT_FDTABLE \
 {							\
@@ -131,5 +132,7 @@ extern struct group_info init_groups;
 	LIST_HEAD_INIT(cpu_timers[2]),					\
 }
 
+#define INIT_CONTAINER(cont)						\
+	.refcnt		= ATOMIC_INIT(1)
 
 #endif
include/linux/sched.h renamed from include/linux/sched.h.vpsinfo +6 -0
@@ -687,6 +687,7 @@ static inline void prefetch_stack(struct
 
 struct audit_context;		/* See audit.c */
 struct mempolicy;
+struct container;
 
 struct task_struct {
 	volatile long state;	/* -1 unrunnable, 0 runnable, >0 stopped */
@@ -846,6 +847,11 @@ struct task_struct {
 
 	struct io_context *io_context;
 
+#ifdef CONFIG_CONTAINER
+	struct container *container;
+	struct container *econtainer;	/* effective container */
+#endif
+
 	unsigned long ptrace_message;
 	siginfo_t *last_siginfo; /* For ptrace use.  */
 /*
init/main.c renamed from init/main.c.vpsinfo +3 -0
@@ -47,6 +47,7 @@
 #include <linux/rmap.h>
 #include <linux/mempolicy.h>
 #include <linux/key.h>
+#include <linux/container.h>
 
 #include <asm/io.h>
 #include <asm/bugs.h>
@@ -603,6 +604,8 @@ static void __init do_initcalls(void)
  */
 static void __init do_basic_setup(void)
 {
+	init_containers();
+
 	/* drivers will send hotplug events */
 	init_workqueues();
 	usermodehelper_init();
kernel/Makefile renamed from kernel/Makefile.vpsinfo +1 -0
@@ -34,6 +34,7 @@ obj-$(CONFIG_DETECT_SOFTLOCKUP) += softl
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
+obj-$(CONFIG_CONTAINER) += container.c
 
 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
 # According to Alan Modra <[email protected]>, the -fno-omit-frame-pointer is
kernel/container.c renamed from kernel/container.c.vpsinfo +16 -0
@@ -0,0 +1,16 @@
+#include <linux/container.h>
+
+/*
+ * Initial container.
+ *
+ * All tasks and other resources initially belong to it
+ */
+struct container init_container = INIT_CONTAINER(init_container);
+
+EXPORT_SYMBOL(init_container);
+
+void init_containers(void)
+{
+	/* remember who is init in container */
+	init_container.init_task = child_reaper;
+}
kernel/exit.c renamed from kernel/exit.c.vpsinfo +2 -0
@@ -31,6 +31,7 @@
 #include <linux/signal.h>
 #include <linux/cn_proc.h>
 #include <linux/mutex.h>
+#include <linux/container.h>
 
 #include <asm/uaccess.h>
 #include <asm/unistd.h>
@@ -107,6 +108,7 @@ repeat: 
 	spin_unlock(&p->proc_lock);
 	proc_pid_flush(proc_dentry);
 	release_thread(p);
+	put_container(p->container);
 	put_task_struct(p);
 
 	p = leader;
kernel/fork.c renamed from kernel/fork.c.vpsinfo +2 -0
@@ -44,6 +44,7 @@
 #include <linux/rmap.h>
 #include <linux/acct.h>
 #include <linux/cn_proc.h>
+#include <linux/container.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -1132,6 +1133,7 @@ static task_t *copy_process(unsigned lon
 	p->ioprio = current->ioprio;
 
 	SET_LINKS(p);
+	(void)get_container(p->container);
 	if (unlikely(p->ptrace & PT_PTRACED))
 		__ptrace_link(p, current->parent);
 
#1 lore raw

PATCH 2/4 Virtualization/containers: CONFIG_CONTAINER

This patch simply adds CONFIG_CONTAINER option for virtualization/containerss functionality. Per-resource config options can be added later if needed.

Kirill

arch/alpha/Kconfig renamed from arch/alpha/Kconfig.vkconfig +2 -0
@@ -621,6 +621,8 @@ source "arch/alpha/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/arm/Kconfig renamed from arch/arm/Kconfig.vkconfig +2 -0
@@ -794,6 +794,8 @@ source "arch/arm/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/arm26/Kconfig renamed from arch/arm26/Kconfig.vkconfig +2 -0
@@ -232,6 +232,8 @@ source "arch/arm26/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/cris/Kconfig renamed from arch/cris/Kconfig.vkconfig +2 -0
@@ -175,6 +175,8 @@ source "arch/cris/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/frv/Kconfig renamed from arch/frv/Kconfig.vkconfig +2 -0
@@ -341,6 +341,8 @@ source "arch/frv/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/h8300/Kconfig renamed from arch/h8300/Kconfig.vkconfig +2 -0
@@ -189,6 +189,8 @@ source "arch/h8300/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/i386/Kconfig renamed from arch/i386/Kconfig.vkconfig +2 -0
@@ -1072,6 +1072,8 @@ source "arch/i386/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/ia64/Kconfig renamed from arch/ia64/Kconfig.vkconfig +2 -0
@@ -463,4 +463,6 @@ source "arch/ia64/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
arch/m32r/Kconfig renamed from arch/m32r/Kconfig.vkconfig +2 -0
@@ -377,6 +377,8 @@ source "arch/m32r/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/m68k/Kconfig renamed from arch/m68k/Kconfig.vkconfig +2 -0
@@ -648,6 +648,8 @@ source "arch/m68k/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/m68knommu/Kconfig renamed from arch/m68knommu/Kconfig.vkconfig +2 -0
@@ -644,6 +644,8 @@ source "arch/m68knommu/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/mips/Kconfig renamed from arch/mips/Kconfig.vkconfig +2 -0
@@ -1811,6 +1811,8 @@ source "arch/mips/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/parisc/Kconfig renamed from arch/parisc/Kconfig.vkconfig +2 -0
@@ -210,6 +210,8 @@ source "arch/parisc/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/powerpc/Kconfig renamed from arch/powerpc/Kconfig.vkconfig +2 -0
@@ -975,4 +975,6 @@ config KEYS_COMPAT
 	depends on COMPAT && KEYS
 	default y
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
arch/ppc/Kconfig renamed from arch/ppc/Kconfig.vkconfig +2 -0
@@ -1396,4 +1396,6 @@ source "arch/ppc/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
arch/s390/Kconfig renamed from arch/s390/Kconfig.vkconfig +2 -0
@@ -470,6 +470,8 @@ source "arch/s390/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/sh/Kconfig renamed from arch/sh/Kconfig.vkconfig +2 -0
@@ -635,6 +635,8 @@ source "arch/sh/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/sh64/Kconfig renamed from arch/sh64/Kconfig.vkconfig +2 -0
@@ -272,6 +272,8 @@ source "arch/sh64/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/sparc/Kconfig renamed from arch/sparc/Kconfig.vkconfig +2 -0
@@ -286,6 +286,8 @@ source "arch/sparc/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/sparc64/Kconfig renamed from arch/sparc64/Kconfig.vkconfig +2 -0
@@ -395,6 +395,8 @@ source "arch/sparc64/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/um/Kconfig renamed from arch/um/Kconfig.vkconfig +2 -0
@@ -292,6 +292,8 @@ source "fs/Kconfig"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/v850/Kconfig renamed from arch/v850/Kconfig.vkconfig +2 -0
@@ -318,6 +318,8 @@ source "arch/v850/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/x86_64/Kconfig renamed from arch/x86_64/Kconfig.vkconfig +2 -0
@@ -604,6 +604,8 @@ source "arch/x86_64/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
arch/xtensa/Kconfig renamed from arch/xtensa/Kconfig.vkconfig +2 -0
@@ -247,6 +247,8 @@ source "arch/xtensa/Kconfig.debug"
 
 source "security/Kconfig"
 
+source "kernel/Kconfig.container"
+
 source "crypto/Kconfig"
 
 source "lib/Kconfig"
kernel/Kconfig.container new file +11 -0
@@ -0,0 +1,11 @@
+menu "Virtual Containers"
+
+config CONTAINER
+	bool "Virtual Containers support"
+	default n
+	help
+	  This option enables support of virtual linux containers,
+	  which can be used for creation of virtual environments,
+	  Virtual Private Servers, checkpointing, isolation and so on
+
+endmenu
#2 lore raw

PATCH 3/4 Virtualization/containers: UID hash

In reply to Kirill Korotaev

This patch virtualizes UID hash, so that processes in container can use it's own UID set. Can be done as an option if some virtualization solutions do not require it.

Kirill

include/linux/container.h renamed from include/linux/container.h.uids +3 -0
@@ -6,11 +6,14 @@
 #include <asm/atomic.h>
 
 struct task_struct;
+struct list_head;
 
 struct container {
 	u32 id;
 	struct task_struct *init_task;
 	atomic_t refcnt;
+
+	struct list_head *c_uid_hash;
 };
 
 extern struct container init_container;
kernel/user.c renamed from kernel/user.c.uids +9 -0
@@ -14,6 +14,7 @@
 #include <linux/bitops.h>
 #include <linux/key.h>
 #include <linux/interrupt.h>
+#include <linux/container.h>
 
 /*
  * UID task count cache, to get fast user lookup in "alloc_uid"
@@ -24,7 +25,12 @@
 #define UIDHASH_SZ		(1 << UIDHASH_BITS)
 #define UIDHASH_MASK		(UIDHASH_SZ - 1)
 #define __uidhashfn(uid)	(((uid >> UIDHASH_BITS) + uid) & UIDHASH_MASK)
+
+#ifdef CONFIG_CONTAINER
+#define uidhashentry(uid)	(econtainer()->c_uid_hash + __uidhashfn((uid)))
+#else
 #define uidhashentry(uid)	(uidhash_table + __uidhashfn((uid)))
+#endif
 
 static kmem_cache_t *uid_cachep;
 static struct list_head uidhash_table[UIDHASH_SZ];
@@ -200,6 +206,9 @@ static int __init uid_cache_init(void)
 
 	/* Insert the root user immediately (init already runs as root) */
 	spin_lock_irq(&uidhash_lock);
+#ifdef CONFIG_CONTAINER
+	init_container.c_uid_hash = uidhash_table;
+#endif
 	uid_hash_insert(&root_user, uidhashentry(0));
 	spin_unlock_irq(&uidhash_lock);
 
#3 lore raw

PATCH 4/4 Virtualization/containers: uts name

In reply to Kirill Korotaev

This patch virtualizes uts name.
main changes are done in container.h, uts_name.h,
all other places are just replacement of system_utsname with uts_name.

Kirill

arch/alpha/kernel/osf_sys.c renamed from arch/alpha/kernel/osf_sys.c.utsnamex +12 -12
@@ -402,15 +402,15 @@ osf_utsname(char __user *name)
 
 	down_read(&uts_sem);
 	error = -EFAULT;
-	if (copy_to_user(name + 0, system_utsname.sysname, 32))
+	if (copy_to_user(name + 0, uts_name.sysname, 32))
 		goto out;
-	if (copy_to_user(name + 32, system_utsname.nodename, 32))
+	if (copy_to_user(name + 32, uts_name.nodename, 32))
 		goto out;
-	if (copy_to_user(name + 64, system_utsname.release, 32))
+	if (copy_to_user(name + 64, uts_name.release, 32))
 		goto out;
-	if (copy_to_user(name + 96, system_utsname.version, 32))
+	if (copy_to_user(name + 96, uts_name.version, 32))
 		goto out;
-	if (copy_to_user(name + 128, system_utsname.machine, 32))
+	if (copy_to_user(name + 128, uts_name.machine, 32))
 		goto out;
 
 	error = 0;
@@ -449,8 +449,8 @@ osf_getdomainname(char __user *name, int
 
 	down_read(&uts_sem);
 	for (i = 0; i < len; ++i) {
-		__put_user(system_utsname.domainname[i], name + i);
-		if (system_utsname.domainname[i] == '\0')
+		__put_user(uts_name.domainname[i], name + i);
+		if (uts_name.domainname[i] == '\0')
 			break;
 	}
 	up_read(&uts_sem);
@@ -608,11 +608,11 @@ asmlinkage long
 osf_sysinfo(int command, char __user *buf, long count)
 {
 	static char * sysinfo_table[] = {
-		system_utsname.sysname,
-		system_utsname.nodename,
-		system_utsname.release,
-		system_utsname.version,
-		system_utsname.machine,
+		uts_name.sysname,
+		uts_name.nodename,
+		uts_name.release,
+		uts_name.version,
+		uts_name.machine,
 		"alpha",	/* instruction set architecture */
 		"dummy",	/* hardware serial number */
 		"dummy",	/* hardware manufacturer */
arch/i386/kernel/sys_i386.c renamed from arch/i386/kernel/sys_i386.c.utsnamex +6 -6
@@ -217,7 +217,7 @@ asmlinkage int sys_uname(struct old_utsn
 	if (!name)
 		return -EFAULT;
 	down_read(&uts_sem);
-	err=copy_to_user(name, &system_utsname, sizeof (*name));
+	err=copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	return err?-EFAULT:0;
 }
@@ -233,15 +233,15 @@ asmlinkage int sys_olduname(struct oldol
   
   	down_read(&uts_sem);
 	
-	error = __copy_to_user(&name->sysname,&system_utsname.sysname,__OLD_UTS_LEN);
+	error = __copy_to_user(&name->sysname,&uts_name.sysname,__OLD_UTS_LEN);
 	error |= __put_user(0,name->sysname+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->nodename,&system_utsname.nodename,__OLD_UTS_LEN);
+	error |= __copy_to_user(&name->nodename,&uts_name.nodename,__OLD_UTS_LEN);
 	error |= __put_user(0,name->nodename+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->release,&system_utsname.release,__OLD_UTS_LEN);
+	error |= __copy_to_user(&name->release,&uts_name.release,__OLD_UTS_LEN);
 	error |= __put_user(0,name->release+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->version,&system_utsname.version,__OLD_UTS_LEN);
+	error |= __copy_to_user(&name->version,&uts_name.version,__OLD_UTS_LEN);
 	error |= __put_user(0,name->version+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->machine,&system_utsname.machine,__OLD_UTS_LEN);
+	error |= __copy_to_user(&name->machine,&uts_name.machine,__OLD_UTS_LEN);
 	error |= __put_user(0,name->machine+__OLD_UTS_LEN);
 	
 	up_read(&uts_sem);
arch/m32r/kernel/sys_m32r.c renamed from arch/m32r/kernel/sys_m32r.c.utsnamex +1 -1
@@ -199,7 +199,7 @@ asmlinkage int sys_uname(struct old_utsn
 	if (!name)
 		return -EFAULT;
 	down_read(&uts_sem);
-	err=copy_to_user(name, &system_utsname, sizeof (*name));
+	err=copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	return err?-EFAULT:0;
 }
arch/mips/kernel/linux32.c renamed from arch/mips/kernel/linux32.c.utsnamex +1 -1
@@ -1150,7 +1150,7 @@ asmlinkage long sys32_newuname(struct ne
 	int ret = 0;
 
 	down_read(&uts_sem);
-	if (copy_to_user(name,&system_utsname,sizeof *name))
+	if (copy_to_user(name,&uts_name,sizeof *name))
 		ret = -EFAULT;
 	up_read(&uts_sem);
 
arch/mips/kernel/syscall.c renamed from arch/mips/kernel/syscall.c.utsnamex +9 -9
@@ -229,7 +229,7 @@ out:
  */
 asmlinkage int sys_uname(struct old_utsname * name)
 {
-	if (name && !copy_to_user(name, &system_utsname, sizeof (*name)))
+	if (name && !copy_to_user(name, &uts_name, sizeof (*name)))
 		return 0;
 	return -EFAULT;
 }
@@ -246,15 +246,15 @@ asmlinkage int sys_olduname(struct oldol
 	if (!access_ok(VERIFY_WRITE,name,sizeof(struct oldold_utsname)))
 		return -EFAULT;
 
-	error = __copy_to_user(&name->sysname,&system_utsname.sysname,__OLD_UTS_LEN);
+	error = __copy_to_user(&name->sysname,&uts_name.sysname,__OLD_UTS_LEN);
 	error -= __put_user(0,name->sysname+__OLD_UTS_LEN);
-	error -= __copy_to_user(&name->nodename,&system_utsname.nodename,__OLD_UTS_LEN);
+	error -= __copy_to_user(&name->nodename,&uts_name.nodename,__OLD_UTS_LEN);
 	error -= __put_user(0,name->nodename+__OLD_UTS_LEN);
-	error -= __copy_to_user(&name->release,&system_utsname.release,__OLD_UTS_LEN);
+	error -= __copy_to_user(&name->release,&uts_name.release,__OLD_UTS_LEN);
 	error -= __put_user(0,name->release+__OLD_UTS_LEN);
-	error -= __copy_to_user(&name->version,&system_utsname.version,__OLD_UTS_LEN);
+	error -= __copy_to_user(&name->version,&uts_name.version,__OLD_UTS_LEN);
 	error -= __put_user(0,name->version+__OLD_UTS_LEN);
-	error -= __copy_to_user(&name->machine,&system_utsname.machine,__OLD_UTS_LEN);
+	error -= __copy_to_user(&name->machine,&uts_name.machine,__OLD_UTS_LEN);
 	error = __put_user(0,name->machine+__OLD_UTS_LEN);
 	error = error ? -EFAULT : 0;
 
@@ -290,10 +290,10 @@ asmlinkage int _sys_sysmips(int cmd, lon
 			return -EFAULT;
 
 		down_write(&uts_sem);
-		strncpy(system_utsname.nodename, nodename, len);
+		strncpy(uts_name.nodename, nodename, len);
 		nodename[__NEW_UTS_LEN] = '\0';
-		strlcpy(system_utsname.nodename, nodename,
-		        sizeof(system_utsname.nodename));
+		strlcpy(uts_name.nodename, nodename,
+		        sizeof(uts_name.nodename));
 		up_write(&uts_sem);
 		return 0;
 	}
arch/mips/kernel/sysirix.c renamed from arch/mips/kernel/sysirix.c.utsnamex +6 -6
@@ -904,7 +904,7 @@ asmlinkage int irix_getdomainname(char _
 	down_read(&uts_sem);
 	if (len > __NEW_UTS_LEN)
 		len = __NEW_UTS_LEN;
-	err = copy_to_user(name, system_utsname.domainname, len) ? -EFAULT : 0;
+	err = copy_to_user(name, uts_name.domainname, len) ? -EFAULT : 0;
 	up_read(&uts_sem);
 
 	return err;
@@ -1147,11 +1147,11 @@ struct iuname {
 asmlinkage int irix_uname(struct iuname __user *buf)
 {
 	down_read(&uts_sem);
-	if (copy_from_user(system_utsname.sysname, buf->sysname, 65)
-	    || copy_from_user(system_utsname.nodename, buf->nodename, 65)
-	    || copy_from_user(system_utsname.release, buf->release, 65)
-	    || copy_from_user(system_utsname.version, buf->version, 65)
-	    || copy_from_user(system_utsname.machine, buf->machine, 65)) {
+	if (copy_from_user(uts_name.sysname, buf->sysname, 65)
+	    || copy_from_user(uts_name.nodename, buf->nodename, 65)
+	    || copy_from_user(uts_name.release, buf->release, 65)
+	    || copy_from_user(uts_name.version, buf->version, 65)
+	    || copy_from_user(uts_name.machine, buf->machine, 65)) {
 		return -EFAULT;
 	}
 	up_read(&uts_sem);
arch/parisc/hpux/sys_hpux.c renamed from arch/parisc/hpux/sys_hpux.c.utsnamex +11 -11
@@ -266,15 +266,15 @@ static int hpux_uname(struct hpux_utsnam
 
 	down_read(&uts_sem);
 
-	error = __copy_to_user(&name->sysname,&system_utsname.sysname,HPUX_UTSLEN-1);
+	error = __copy_to_user(&name->sysname,&uts_name.sysname,HPUX_UTSLEN-1);
 	error |= __put_user(0,name->sysname+HPUX_UTSLEN-1);
-	error |= __copy_to_user(&name->nodename,&system_utsname.nodename,HPUX_UTSLEN-1);
+	error |= __copy_to_user(&name->nodename,&uts_name.nodename,HPUX_UTSLEN-1);
 	error |= __put_user(0,name->nodename+HPUX_UTSLEN-1);
-	error |= __copy_to_user(&name->release,&system_utsname.release,HPUX_UTSLEN-1);
+	error |= __copy_to_user(&name->release,&uts_name.release,HPUX_UTSLEN-1);
 	error |= __put_user(0,name->release+HPUX_UTSLEN-1);
-	error |= __copy_to_user(&name->version,&system_utsname.version,HPUX_UTSLEN-1);
+	error |= __copy_to_user(&name->version,&uts_name.version,HPUX_UTSLEN-1);
 	error |= __put_user(0,name->version+HPUX_UTSLEN-1);
-	error |= __copy_to_user(&name->machine,&system_utsname.machine,HPUX_UTSLEN-1);
+	error |= __copy_to_user(&name->machine,&uts_name.machine,HPUX_UTSLEN-1);
 	error |= __put_user(0,name->machine+HPUX_UTSLEN-1);
 
 	up_read(&uts_sem);
@@ -373,8 +373,8 @@ int hpux_utssys(char *ubuf, int n, int t
 		/*  TODO:  print a warning about using this?  */
 		down_write(&uts_sem);
 		error = -EFAULT;
-		if (!copy_from_user(system_utsname.sysname, ubuf, len)) {
-			system_utsname.sysname[len] = 0;
+		if (!copy_from_user(uts_name.sysname, ubuf, len)) {
+			uts_name.sysname[len] = 0;
 			error = 0;
 		}
 		up_write(&uts_sem);
@@ -400,8 +400,8 @@ int hpux_utssys(char *ubuf, int n, int t
 		/*  TODO:  print a warning about this?  */
 		down_write(&uts_sem);
 		error = -EFAULT;
-		if (!copy_from_user(system_utsname.release, ubuf, len)) {
-			system_utsname.release[len] = 0;
+		if (!copy_from_user(uts_name.release, ubuf, len)) {
+			uts_name.release[len] = 0;
 			error = 0;
 		}
 		up_write(&uts_sem);
@@ -422,13 +422,13 @@ int hpux_getdomainname(char *name, int l
  	
  	down_read(&uts_sem);
  	
-	nlen = strlen(system_utsname.domainname) + 1;
+	nlen = strlen(uts_name.domainname) + 1;
 
 	if (nlen < len)
 		len = nlen;
 	if(len > __NEW_UTS_LEN)
 		goto done;
-	if(copy_to_user(name, system_utsname.domainname, len))
+	if(copy_to_user(name, uts_name.domainname, len))
 		goto done;
 	err = 0;
 done:
arch/powerpc/kernel/syscalls.c renamed from arch/powerpc/kernel/syscalls.c.utsnamex +7 -7
@@ -259,7 +259,7 @@ long ppc_newuname(struct new_utsname __u
 	int err = 0;
 
 	down_read(&uts_sem);
-	if (copy_to_user(name, &system_utsname, sizeof(*name)))
+	if (copy_to_user(name, &uts_name, sizeof(*name)))
 		err = -EFAULT;
 	up_read(&uts_sem);
 	if (!err)
@@ -272,7 +272,7 @@ int sys_uname(struct old_utsname __user 
 	int err = 0;
 	
 	down_read(&uts_sem);
-	if (copy_to_user(name, &system_utsname, sizeof(*name)))
+	if (copy_to_user(name, &uts_name, sizeof(*name)))
 		err = -EFAULT;
 	up_read(&uts_sem);
 	if (!err)
@@ -288,19 +288,19 @@ int sys_olduname(struct oldold_utsname _
 		return -EFAULT;
   
 	down_read(&uts_sem);
-	error = __copy_to_user(&name->sysname, &system_utsname.sysname,
+	error = __copy_to_user(&name->sysname, &uts_name.sysname,
 			       __OLD_UTS_LEN);
 	error |= __put_user(0, name->sysname + __OLD_UTS_LEN);
-	error |= __copy_to_user(&name->nodename, &system_utsname.nodename,
+	error |= __copy_to_user(&name->nodename, &uts_name.nodename,
 				__OLD_UTS_LEN);
 	error |= __put_user(0, name->nodename + __OLD_UTS_LEN);
-	error |= __copy_to_user(&name->release, &system_utsname.release,
+	error |= __copy_to_user(&name->release, &uts_name.release,
 				__OLD_UTS_LEN);
 	error |= __put_user(0, name->release + __OLD_UTS_LEN);
-	error |= __copy_to_user(&name->version, &system_utsname.version,
+	error |= __copy_to_user(&name->version, &uts_name.version,
 				__OLD_UTS_LEN);
 	error |= __put_user(0, name->version + __OLD_UTS_LEN);
-	error |= __copy_to_user(&name->machine, &system_utsname.machine,
+	error |= __copy_to_user(&name->machine, &uts_name.machine,
 				__OLD_UTS_LEN);
 	error |= override_machine(name->machine);
 	up_read(&uts_sem);
arch/sh/kernel/setup.c renamed from arch/sh/kernel/setup.c.utsnamex +1 -1
@@ -485,7 +485,7 @@ static int show_cpuinfo(struct seq_file 
 		seq_printf(m, "machine\t\t: %s\n", get_system_type());
 
 	seq_printf(m, "processor\t: %d\n", cpu);
-	seq_printf(m, "cpu family\t: %s\n", system_utsname.machine);
+	seq_printf(m, "cpu family\t: %s\n", uts_name.machine);
 	seq_printf(m, "cpu type\t: %s\n", get_cpu_subtype());
 
 	show_cpuflags(m);
arch/sh/kernel/sys_sh.c renamed from arch/sh/kernel/sys_sh.c.utsnamex +1 -1
@@ -267,7 +267,7 @@ asmlinkage int sys_uname(struct old_utsn
 	if (!name)
 		return -EFAULT;
 	down_read(&uts_sem);
-	err=copy_to_user(name, &system_utsname, sizeof (*name));
+	err=copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	return err?-EFAULT:0;
 }
arch/sh64/kernel/sys_sh64.c renamed from arch/sh64/kernel/sys_sh64.c.utsnamex +1 -1
@@ -279,7 +279,7 @@ asmlinkage int sys_uname(struct old_utsn
 	if (!name)
 		return -EFAULT;
 	down_read(&uts_sem);
-	err=copy_to_user(name, &system_utsname, sizeof (*name));
+	err=copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	return err?-EFAULT:0;
 }
arch/sparc/kernel/sys_sparc.c renamed from arch/sparc/kernel/sys_sparc.c.utsnamex +2 -2
@@ -470,13 +470,13 @@ asmlinkage int sys_getdomainname(char __
  	
  	down_read(&uts_sem);
  	
-	nlen = strlen(system_utsname.domainname) + 1;
+	nlen = strlen(uts_name.domainname) + 1;
 
 	if (nlen < len)
 		len = nlen;
 	if (len > __NEW_UTS_LEN)
 		goto done;
-	if (copy_to_user(name, system_utsname.domainname, len))
+	if (copy_to_user(name, uts_name.domainname, len))
 		goto done;
 	err = 0;
 done:
arch/sparc/kernel/sys_sunos.c renamed from arch/sparc/kernel/sys_sunos.c.utsnamex +5 -5
@@ -483,13 +483,13 @@ asmlinkage int sunos_uname(struct sunos_
 {
 	int ret;
 	down_read(&uts_sem);
-	ret = copy_to_user(&name->sname[0], &system_utsname.sysname[0], sizeof(name->sname) - 1);
+	ret = copy_to_user(&name->sname[0], &uts_name.sysname[0], sizeof(name->sname) - 1);
 	if (!ret) {
-		ret |= __copy_to_user(&name->nname[0], &system_utsname.nodename[0], sizeof(name->nname) - 1);
+		ret |= __copy_to_user(&name->nname[0], &uts_name.nodename[0], sizeof(name->nname) - 1);
 		ret |= __put_user('\0', &name->nname[8]);
-		ret |= __copy_to_user(&name->rel[0], &system_utsname.release[0], sizeof(name->rel) - 1);
-		ret |= __copy_to_user(&name->ver[0], &system_utsname.version[0], sizeof(name->ver) - 1);
-		ret |= __copy_to_user(&name->mach[0], &system_utsname.machine[0], sizeof(name->mach) - 1);
+		ret |= __copy_to_user(&name->rel[0], &uts_name.release[0], sizeof(name->rel) - 1);
+		ret |= __copy_to_user(&name->ver[0], &uts_name.version[0], sizeof(name->ver) - 1);
+		ret |= __copy_to_user(&name->mach[0], &uts_name.machine[0], sizeof(name->mach) - 1);
 	}
 	up_read(&uts_sem);
 	return ret ? -EFAULT : 0;
arch/sparc64/kernel/sys_sparc.c renamed from arch/sparc64/kernel/sys_sparc.c.utsnamex +2 -2
@@ -476,13 +476,13 @@ asmlinkage long sys_getdomainname(char _
 
  	down_read(&uts_sem);
  	
-	nlen = strlen(system_utsname.domainname) + 1;
+	nlen = strlen(uts_name.domainname) + 1;
 
         if (nlen < len)
                 len = nlen;
 	if (len > __NEW_UTS_LEN)
 		goto done;
-	if (copy_to_user(name, system_utsname.domainname, len))
+	if (copy_to_user(name, uts_name.domainname, len))
 		goto done;
 	err = 0;
 done:
arch/sparc64/kernel/sys_sunos32.c renamed from arch/sparc64/kernel/sys_sunos32.c.utsnamex +5 -5
@@ -439,16 +439,16 @@ asmlinkage int sunos_uname(struct sunos_
 	int ret;
 
 	down_read(&uts_sem);
-	ret = copy_to_user(&name->sname[0], &system_utsname.sysname[0],
+	ret = copy_to_user(&name->sname[0], &uts_name.sysname[0],
 			   sizeof(name->sname) - 1);
-	ret |= copy_to_user(&name->nname[0], &system_utsname.nodename[0],
+	ret |= copy_to_user(&name->nname[0], &uts_name.nodename[0],
 			    sizeof(name->nname) - 1);
 	ret |= put_user('\0', &name->nname[8]);
-	ret |= copy_to_user(&name->rel[0], &system_utsname.release[0],
+	ret |= copy_to_user(&name->rel[0], &uts_name.release[0],
 			    sizeof(name->rel) - 1);
-	ret |= copy_to_user(&name->ver[0], &system_utsname.version[0],
+	ret |= copy_to_user(&name->ver[0], &uts_name.version[0],
 			    sizeof(name->ver) - 1);
-	ret |= copy_to_user(&name->mach[0], &system_utsname.machine[0],
+	ret |= copy_to_user(&name->mach[0], &uts_name.machine[0],
 			    sizeof(name->mach) - 1);
 	up_read(&uts_sem);
 	return (ret ? -EFAULT : 0);
arch/sparc64/solaris/misc.c renamed from arch/sparc64/solaris/misc.c.utsnamex +3 -3
@@ -239,7 +239,7 @@ asmlinkage int solaris_utssys(u32 buf, u
 		/* Let's cheat */
 		err  = set_utsfield(v->sysname, "SunOS", 1, 0);
 		down_read(&uts_sem);
-		err |= set_utsfield(v->nodename, system_utsname.nodename,
+		err |= set_utsfield(v->nodename, uts_name.nodename,
 				    1, 1);
 		up_read(&uts_sem);
 		err |= set_utsfield(v->release, "2.6", 0, 0);
@@ -263,7 +263,7 @@ asmlinkage int solaris_utsname(u32 buf)
 	/* Why should we not lie a bit? */
 	down_read(&uts_sem);
 	err  = set_utsfield(v->sysname, "SunOS", 0, 0);
-	err |= set_utsfield(v->nodename, system_utsname.nodename, 1, 1);
+	err |= set_utsfield(v->nodename, uts_name.nodename, 1, 1);
 	err |= set_utsfield(v->release, "5.6", 0, 0);
 	err |= set_utsfield(v->version, "Generic", 0, 0);
 	err |= set_utsfield(v->machine, machine(), 0, 0);
@@ -295,7 +295,7 @@ asmlinkage int solaris_sysinfo(int cmd, 
 	case SI_HOSTNAME:
 		r = buffer + 256;
 		down_read(&uts_sem);
-		for (p = system_utsname.nodename, q = buffer; 
+		for (p = uts_name.nodename, q = buffer; 
 		     q < r && *p && *p != '.'; *q++ = *p++);
 		up_read(&uts_sem);
 		*q = 0;
arch/um/kernel/syscall_kern.c renamed from arch/um/kernel/syscall_kern.c.utsnamex +6 -6
@@ -110,7 +110,7 @@ long sys_uname(struct old_utsname * name
 	if (!name)
 		return -EFAULT;
 	down_read(&uts_sem);
-	err=copy_to_user(name, &system_utsname, sizeof (*name));
+	err=copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	return err?-EFAULT:0;
 }
@@ -126,19 +126,19 @@ long sys_olduname(struct oldold_utsname 
   
   	down_read(&uts_sem);
 	
-	error = __copy_to_user(&name->sysname,&system_utsname.sysname,
+	error = __copy_to_user(&name->sysname,&uts_name.sysname,
 			       __OLD_UTS_LEN);
 	error |= __put_user(0,name->sysname+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->nodename,&system_utsname.nodename,
+	error |= __copy_to_user(&name->nodename,&uts_name.nodename,
 				__OLD_UTS_LEN);
 	error |= __put_user(0,name->nodename+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->release,&system_utsname.release,
+	error |= __copy_to_user(&name->release,&uts_name.release,
 				__OLD_UTS_LEN);
 	error |= __put_user(0,name->release+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->version,&system_utsname.version,
+	error |= __copy_to_user(&name->version,&uts_name.version,
 				__OLD_UTS_LEN);
 	error |= __put_user(0,name->version+__OLD_UTS_LEN);
-	error |= __copy_to_user(&name->machine,&system_utsname.machine,
+	error |= __copy_to_user(&name->machine,&uts_name.machine,
 				__OLD_UTS_LEN);
 	error |= __put_user(0,name->machine+__OLD_UTS_LEN);
 	
arch/um/sys-x86_64/syscalls.c renamed from arch/um/sys-x86_64/syscalls.c.utsnamex +1 -1
@@ -21,7 +21,7 @@ asmlinkage long sys_uname64(struct new_u
 {
 	int err;
 	down_read(&uts_sem);
-	err = copy_to_user(name, &system_utsname, sizeof (*name));
+	err = copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	if (personality(current->personality) == PER_LINUX32)
 		err |= copy_to_user(&name->machine, "i686", 5);
arch/x86_64/ia32/sys_ia32.c renamed from arch/x86_64/ia32/sys_ia32.c.utsnamex +5 -5
@@ -868,13 +868,13 @@ asmlinkage long sys32_olduname(struct ol
   
   	down_read(&uts_sem);
 	
-	error = __copy_to_user(&name->sysname,&system_utsname.sysname,__OLD_UTS_LEN);
+	error = __copy_to_user(&name->sysname,&uts_name.sysname,__OLD_UTS_LEN);
 	 __put_user(0,name->sysname+__OLD_UTS_LEN);
-	 __copy_to_user(&name->nodename,&system_utsname.nodename,__OLD_UTS_LEN);
+	 __copy_to_user(&name->nodename,&uts_name.nodename,__OLD_UTS_LEN);
 	 __put_user(0,name->nodename+__OLD_UTS_LEN);
-	 __copy_to_user(&name->release,&system_utsname.release,__OLD_UTS_LEN);
+	 __copy_to_user(&name->release,&uts_name.release,__OLD_UTS_LEN);
 	 __put_user(0,name->release+__OLD_UTS_LEN);
-	 __copy_to_user(&name->version,&system_utsname.version,__OLD_UTS_LEN);
+	 __copy_to_user(&name->version,&uts_name.version,__OLD_UTS_LEN);
 	 __put_user(0,name->version+__OLD_UTS_LEN);
 	 { 
 		 char *arch = "x86_64";
@@ -897,7 +897,7 @@ long sys32_uname(struct old_utsname __us
 	if (!name)
 		return -EFAULT;
 	down_read(&uts_sem);
-	err=copy_to_user(name, &system_utsname, sizeof (*name));
+	err=copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	if (personality(current->personality) == PER_LINUX32) 
 		err |= copy_to_user(&name->machine, "i686", 5);
arch/x86_64/kernel/sys_x86_64.c renamed from arch/x86_64/kernel/sys_x86_64.c.utsnamex +1 -1
@@ -148,7 +148,7 @@ asmlinkage long sys_uname(struct new_uts
 {
 	int err;
 	down_read(&uts_sem);
-	err = copy_to_user(name, &system_utsname, sizeof (*name));
+	err = copy_to_user(name, &uts_name, sizeof (*name));
 	up_read(&uts_sem);
 	if (personality(current->personality) == PER_LINUX32) 
 		err |= copy_to_user(&name->machine, "i686", 5); 		
arch/xtensa/kernel/syscalls.c renamed from arch/xtensa/kernel/syscalls.c.utsnamex +1 -1
@@ -129,7 +129,7 @@ out:
 
 int sys_uname(struct old_utsname * name)
 {
-	if (name && !copy_to_user(name, &system_utsname, sizeof (*name)))
+	if (name && !copy_to_user(name, &uts_name, sizeof (*name)))
 		return 0;
 	return -EFAULT;
 }
drivers/usb/core/hcd.c renamed from drivers/usb/core/hcd.c.utsnamex +2 -2
@@ -317,8 +317,8 @@ static int rh_string (
 
  	// id 3 == vendor description
 	} else if (id == 3) {
-		snprintf (buf, sizeof buf, "%s %s %s", system_utsname.sysname,
-			system_utsname.release, hcd->driver->description);
+		snprintf (buf, sizeof buf, "%s %s %s", uts_name.sysname,
+			uts_name.release, hcd->driver->description);
 
 	// unsupported IDs --> "protocol stall"
 	} else
fs/cifs/connect.c renamed from fs/cifs/connect.c.utsnamex +14 -14
@@ -765,12 +765,12 @@ cifs_parse_mount_options(char *options, 
 	separator[1] = 0; 
 
 	memset(vol->source_rfc1001_name,0x20,15);
-	for(i=0;i < strnlen(system_utsname.nodename,15);i++) {
+	for(i=0;i < strnlen(uts_name.nodename,15);i++) {
 		/* does not have to be a perfect mapping since the field is
 		informational, only used for servers that do not support
 		port 445 and it can be overridden at mount time */
 		vol->source_rfc1001_name[i] = 
-			toupper(system_utsname.nodename[i]);
+			toupper(uts_name.nodename[i]);
 	}
 	vol->source_rfc1001_name[15] = 0;
 	/* null target name indicates to use *SMBSERVR default called name
@@ -2062,7 +2062,7 @@ CIFSSessSetup(unsigned int xid, struct c
 				  32, nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bytes_returned =
-		    cifs_strtoUCS((__le16 *) bcc_ptr, system_utsname.release,
+		    cifs_strtoUCS((__le16 *) bcc_ptr, uts_name.release,
 				  32, nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bcc_ptr += 2;
@@ -2089,8 +2089,8 @@ CIFSSessSetup(unsigned int xid, struct c
 		}
 		strcpy(bcc_ptr, "Linux version ");
 		bcc_ptr += strlen("Linux version ");
-		strcpy(bcc_ptr, system_utsname.release);
-		bcc_ptr += strlen(system_utsname.release) + 1;
+		strcpy(bcc_ptr, uts_name.release);
+		bcc_ptr += strlen(uts_name.release) + 1;
 		strcpy(bcc_ptr, CIFS_NETWORK_OPSYS);
 		bcc_ptr += strlen(CIFS_NETWORK_OPSYS) + 1;
 	}
@@ -2329,7 +2329,7 @@ CIFSSpnegoSessSetup(unsigned int xid, st
 				  32, nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bytes_returned =
-		    cifs_strtoUCS((__le16 *) bcc_ptr, system_utsname.release, 32,
+		    cifs_strtoUCS((__le16 *) bcc_ptr, uts_name.release, 32,
 				  nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bcc_ptr += 2;
@@ -2354,8 +2354,8 @@ CIFSSpnegoSessSetup(unsigned int xid, st
 		}
 		strcpy(bcc_ptr, "Linux version ");
 		bcc_ptr += strlen("Linux version ");
-		strcpy(bcc_ptr, system_utsname.release);
-		bcc_ptr += strlen(system_utsname.release) + 1;
+		strcpy(bcc_ptr, uts_name.release);
+		bcc_ptr += strlen(uts_name.release) + 1;
 		strcpy(bcc_ptr, CIFS_NETWORK_OPSYS);
 		bcc_ptr += strlen(CIFS_NETWORK_OPSYS) + 1;
 	}
@@ -2619,7 +2619,7 @@ CIFSNTLMSSPNegotiateSessSetup(unsigned i
 				  32, nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bytes_returned =
-		    cifs_strtoUCS((__le16 *) bcc_ptr, system_utsname.release, 32,
+		    cifs_strtoUCS((__le16 *) bcc_ptr, uts_name.release, 32,
 				  nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bcc_ptr += 2;	/* null terminate Linux version */
@@ -2636,8 +2636,8 @@ CIFSNTLMSSPNegotiateSessSetup(unsigned i
 	} else {		/* ASCII */
 		strcpy(bcc_ptr, "Linux version ");
 		bcc_ptr += strlen("Linux version ");
-		strcpy(bcc_ptr, system_utsname.release);
-		bcc_ptr += strlen(system_utsname.release) + 1;
+		strcpy(bcc_ptr, uts_name.release);
+		bcc_ptr += strlen(uts_name.release) + 1;
 		strcpy(bcc_ptr, CIFS_NETWORK_OPSYS);
 		bcc_ptr += strlen(CIFS_NETWORK_OPSYS) + 1;
 		bcc_ptr++;	/* empty domain field */
@@ -2998,7 +2998,7 @@ CIFSNTLMSSPAuthSessSetup(unsigned int xi
 				  32, nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bytes_returned =
-		    cifs_strtoUCS((__le16 *) bcc_ptr, system_utsname.release, 32,
+		    cifs_strtoUCS((__le16 *) bcc_ptr, uts_name.release, 32,
 				  nls_codepage);
 		bcc_ptr += 2 * bytes_returned;
 		bcc_ptr += 2;	/* null term version string */
@@ -3050,8 +3050,8 @@ CIFSNTLMSSPAuthSessSetup(unsigned int xi
 
 		strcpy(bcc_ptr, "Linux version ");
 		bcc_ptr += strlen("Linux version ");
-		strcpy(bcc_ptr, system_utsname.release);
-		bcc_ptr += strlen(system_utsname.release) + 1;
+		strcpy(bcc_ptr, uts_name.release);
+		bcc_ptr += strlen(uts_name.release) + 1;
 		strcpy(bcc_ptr, CIFS_NETWORK_OPSYS);
 		bcc_ptr += strlen(CIFS_NETWORK_OPSYS) + 1;
 		bcc_ptr++;	/* null domain */
fs/exec.c renamed from fs/exec.c.utsnamex +1 -1
@@ -1326,7 +1326,7 @@ static void format_corename(char *corena
 			case 'h':
 				down_read(&uts_sem);
 				rc = snprintf(out_ptr, out_end - out_ptr,
-					      "%s", system_utsname.nodename);
+					      "%s", uts_name.nodename);
 				up_read(&uts_sem);
 				if (rc > out_end - out_ptr)
 					goto out;
fs/lockd/clntproc.c renamed from fs/lockd/clntproc.c.utsnamex +3 -3
@@ -130,10 +130,10 @@ static void nlmclnt_setlockargs(struct n
 	nlmclnt_next_cookie(&argp->cookie);
 	argp->state   = nsm_local_state;
 	memcpy(&lock->fh, NFS_FH(fl->fl_file->f_dentry->d_inode), sizeof(struct nfs_fh));
-	lock->caller  = system_utsname.nodename;
+	lock->caller  = uts_name.nodename;
 	lock->oh.data = req->a_owner;
 	lock->oh.len  = sprintf(req->a_owner, "%d@%s",
-				current->pid, system_utsname.nodename);
+				current->pid, uts_name.nodename);
 	locks_copy_lock(&lock->fl, fl);
 }
 
@@ -154,7 +154,7 @@ nlmclnt_setgrantargs(struct nlm_rqst *ca
 {
 	locks_copy_lock(&call->a_args.lock.fl, &lock->fl);
 	memcpy(&call->a_args.lock.fh, &lock->fh, sizeof(call->a_args.lock.fh));
-	call->a_args.lock.caller = system_utsname.nodename;
+	call->a_args.lock.caller = uts_name.nodename;
 	call->a_args.lock.oh.len = lock->oh.len;
 
 	/* set default data area */
include/asm-i386/elf.h renamed from include/asm-i386/elf.h.utsnamex +1 -1
@@ -108,7 +108,7 @@ typedef struct user_fxsr_struct elf_fpxr
    For the moment, we have only optimizations for the Intel generations,
    but that could change... */
 
-#define ELF_PLATFORM  (system_utsname.machine)
+#define ELF_PLATFORM  (uts_name.machine)
 
 #ifdef __KERNEL__
 #define SET_PERSONALITY(ex, ibcs2) do { } while (0)
include/linux/container.h renamed from include/linux/container.h.utsnamex +2 -0
@@ -7,6 +7,7 @@
 
 struct task_struct;
 struct list_head;
+struct new_utsname;
 
 struct container {
 	u32 id;
@@ -14,6 +15,7 @@ struct container {
 	atomic_t refcnt;
 
 	struct list_head *c_uid_hash;
+	struct new_utsname *c_uts_name;
 };
 
 extern struct container init_container;
include/linux/init_task.h renamed from include/linux/init_task.h.utsnamex +2 -1
@@ -133,6 +133,7 @@ extern struct group_info init_groups;
 }
 
 #define INIT_CONTAINER(cont)						\
-	.refcnt		= ATOMIC_INIT(1)
+	.refcnt		= ATOMIC_INIT(1)				\
+	.c_uts_name	= &system_utsname,
 
 #endif
include/linux/utsname.h renamed from include/linux/utsname.h.utsnamex +7 -0
@@ -1,6 +1,8 @@
 #ifndef _LINUX_UTSNAME_H
 #define _LINUX_UTSNAME_H
 
+#include <linux/container.h>
+
 #define __OLD_UTS_LEN 8
 
 struct oldold_utsname {
@@ -31,6 +33,11 @@ struct new_utsname {
 };
 
 extern struct new_utsname system_utsname;
+#ifdef CONFIG_CONTAINER
+#define uts_name	(*(econtainer()->c_uts_name))
+#else
+#define uts_name	system_utsname
+#endif
 
 extern struct rw_semaphore uts_sem;
 #endif
kernel/sys.c renamed from kernel/sys.c.utsnamex +7 -7
@@ -1518,7 +1518,7 @@ asmlinkage long sys_newuname(struct new_
 	int errno = 0;
 
 	down_read(&uts_sem);
-	if (copy_to_user(name,&system_utsname,sizeof *name))
+	if (copy_to_user(name,&uts_name,sizeof *name))
 		errno = -EFAULT;
 	up_read(&uts_sem);
 	return errno;
@@ -1536,8 +1536,8 @@ asmlinkage long sys_sethostname(char __u
 	down_write(&uts_sem);
 	errno = -EFAULT;
 	if (!copy_from_user(tmp, name, len)) {
-		memcpy(system_utsname.nodename, tmp, len);
-		system_utsname.nodename[len] = 0;
+		memcpy(uts_name.nodename, tmp, len);
+		uts_name.nodename[len] = 0;
 		errno = 0;
 	}
 	up_write(&uts_sem);
@@ -1553,11 +1553,11 @@ asmlinkage long sys_gethostname(char __u
 	if (len < 0)
 		return -EINVAL;
 	down_read(&uts_sem);
-	i = 1 + strlen(system_utsname.nodename);
+	i = 1 + strlen(uts_name.nodename);
 	if (i > len)
 		i = len;
 	errno = 0;
-	if (copy_to_user(name, system_utsname.nodename, i))
+	if (copy_to_user(name, uts_name.nodename, i))
 		errno = -EFAULT;
 	up_read(&uts_sem);
 	return errno;
@@ -1582,8 +1582,8 @@ asmlinkage long sys_setdomainname(char _
 	down_write(&uts_sem);
 	errno = -EFAULT;
 	if (!copy_from_user(tmp, name, len)) {
-		memcpy(system_utsname.domainname, tmp, len);
-		system_utsname.domainname[len] = 0;
+		memcpy(uts_name.domainname, tmp, len);
+		uts_name.domainname[len] = 0;
 		errno = 0;
 	}
 	up_write(&uts_sem);
net/sunrpc/clnt.c renamed from net/sunrpc/clnt.c.utsnamex +2 -2
@@ -168,10 +168,10 @@ rpc_new_client(struct rpc_xprt *xprt, ch
 	}
 
 	/* save the nodename */
-	clnt->cl_nodelen = strlen(system_utsname.nodename);
+	clnt->cl_nodelen = strlen(uts_name.nodename);
 	if (clnt->cl_nodelen > UNX_MAXNODENAME)
 		clnt->cl_nodelen = UNX_MAXNODENAME;
-	memcpy(clnt->cl_nodename, system_utsname.nodename, clnt->cl_nodelen);
+	memcpy(clnt->cl_nodename, uts_name.nodename, clnt->cl_nodelen);
 	return clnt;
 
 out_no_auth:
#4 lore raw
In reply to Kirill Korotaev

kernel/fork.c static task_t *copy_process(unsigned lon · line 1138

Quoted 7 lines · Kirill Korotaev
kernel/fork.c renamed from kernel/fork.c.vpsinfo +1 -0
@@ -1132,6 +1133,7 @@ static task_t *copy_process(unsigned lon
        p->ioprio = current->ioprio;
 
        SET_LINKS(p);
+       (void)get_container(p->container);
        if (unlikely(p->ptrace & PT_PTRACED))
                __ptrace_link(p, current->parent); 

This entire patch looks nice and very straightforward, except for this bit. :) The "(void)" bit isn't usual kernel coding style. You can probably kill it.

BTW, why does get_container() return the container argument? get_task_struct(), for instance is just a do{}while(0) loop, so it doesn't have a return value. Is there some magic later on in your patch set that utilizes this?

One other really minor thing: I usually try to do is keep the ! CONFIG_FOO functions static inlines, just like the full versions. The advantage is that you get some compile-time type checking, even when your CONFIG option is off.

-- Dave

#6 lore raw
In reply to Kirill Korotaev
Quoted 7 lines · Kirill Korotaev

Hello,

I tried to take into account all the comments from you guys (thanks a lot for them!) and prepared a new version of virtualization patches. I will send only 4 patches today, just not to overflow everyone and keep it clear/tidy/possible to review.

Quoted 6 lines

This patch introduces some abstract container kernel structure and a number of operations on it.

The important properties of the proposed container implementation:
- each container has unique ID in the system

What namespace does this ID live in?

Quoted 1 line

- each process in the kernel can belong to one container only

Reasonable.

Quoted 3 lines

- effective container pointer (econtainer()) is used on the task to avoid insertion of additional argument "container" to all functions where it is required.

Why is that desirable?

Quoted 2 lines

- kernel compilation with disabled virtualization should result in old good linux kernel

A reasonable goal.

Why do we need a container structure to hold pointers to other pointers?

Quoted 11 lines

Patches following this one will be used for virtualization of the kernel resources based on this container infrastructure, including those VPID patches I sent before. Every virtualized resource can be given separate config option if needed (just give me to know if it is desired).

Kirill

P.S. I understand that this virtualization spam can be unintersting for some of you, just give me to know if you want to be removed from CC.

May I please be added to the CC list.

We are never going to form a consensus if all of the people doing implementations don't talk.

Eric

#7 lore raw
Quoted 2 lines · Eric W. Biederman

We are never going to form a consensus if all of the people doing implementations don't talk.

Speaking of which - it would be interesting to get Kirill's comments on Eric's patchset ;)

Once we know what's good and bad about both patchsets, we'll be a lot closer to knowing what exactly should go upstream.

Signature · 1 line
All Rights Reversed
#8 lore raw
Quoted 11 lines · Rik van Riel
Quoted 1 line · Eric W. Biederman

We are never going to form a consensus if all of the people doing implementations don't talk.

Speaking of which - it would be interesting to get Kirill's comments on Eric's patchset ;)

Once we know what's good and bad about both patchsets, we'll be a lot closer to knowing what exactly should go upstream.

Let's compare approaches of patchsets before the patchsets themselves.

It seems to be, should we:

A) make a general form of virtualising PIDs, and hope this assists later virtualisation efforts (Eric's patch)

B) make a general form of containers/jails/vservers/vpses, and layer PID virtualisation on top of it somewhere (as in openvz, vserver)

I can't think of any real use cases where you would specifically want A) without B).

Also, the problem space in B) is now very well explored. To start with
A) would mean to throw away 4+ years of experience at this approach (just counting vserver and variants - not FreeBSD Jail, etc). Trying to re-base B) atop a massive refactoring and new patch like A) would incur a lot of work; however fitting it into B) is natural and solved conceptually and in practice, with the only drawback I see being that the use cases mentioned above wouldn't suffer from the side-effects of B).

Perhaps I'm wrong there, but that's my gut feeling.

Sam.

#9 lore raw
Quoted 4 lines
Quoted 4 lines
Quoted 1 line

We are never going to form a consensus if all of the people doing implementations don't talk.

Speaking of which - it would be interesting to get Kirill's comments on Eric's patchset ;)

I'll do comment.

Quoted 2 lines
Quoted 2 lines

Once we know what's good and bad about both patchsets, we'll be a lot closer to knowing what exactly should go upstream.

I'm starting to think that nothing in upstream can be better for all of us :)

Quoted 11 lines

Let's compare approaches of patchsets before the patchsets themselves. It seems to be, should we:

A) make a general form of virtualising PIDs, and hope this assists later virtualisation efforts (Eric's patch)

B) make a general form of containers/jails/vservers/vpses, and layer PID virtualisation on top of it somewhere (as in openvz, vserver)

I can't think of any real use cases where you would specifically want A) without B).

Exactly! All these patches for A) look weird for me without containers itself. A try to make half-solution which is bad.

Quoted 8 lines

Also, the problem space in B) is now very well explored. To start with
A) would mean to throw away 4+ years of experience at this approach (just counting vserver and variants - not FreeBSD Jail, etc). Trying to re-base B) atop a massive refactoring and new patch like A) would incur a lot of work; however fitting it into B) is natural and solved conceptually and in practice, with the only drawback I see being that the use cases mentioned above wouldn't suffer from the side-effects of B).

Have you saw my patches?
This is B) :) This is what we should start with IMHO.
Having a containers and isolation all these talks about A) will be much more precise.

Kirill

#65 lore raw
In reply to Eric W. Biederman
Quoted 3 lines
Quoted 2 lines

The important properties of the proposed container implementation:
- each container has unique ID in the system

What namespace does this ID live in?

global namespace. can be virtualized later.
can be optional.
But the idea is simple. Eventually you will need some management tools anyway. And they should be able to refer to containers.

Quoted 8 lines
Quoted 1 line

- each process in the kernel can belong to one container only

Reasonable.

Quoted 3 lines

- effective container pointer (econtainer()) is used on the task to avoid insertion of additional argument "container" to all functions where it is required.

Why is that desirable?

It was discussed with Linus and the reason is provided in this text actually.
There are 2 ways:
- to add additional argument "container" to all the functions where it is required.
Drawbacks: a) lot's of changes, b) compilation without virtualization is not the same. c) increased stakc usage
- to add effective container pointer on the task. i.e. context which kernel should be in when works with virtualized resources.
Drawbacks: a) there are some places where you need to change effective container context explicitly such as TCP/IP.

Quoted 6 lines
Quoted 2 lines

- kernel compilation with disabled virtualization should result in old good linux kernel

A reasonable goal.

Why do we need a container structure to hold pointers to other pointers?

can't catch what you mean :) is it prohibited somehow? :))))

Quoted 3 lines

May I please be added to the CC list.
We are never going to form a consensus if all of the people doing implementations don't talk.

To make a consensus people need to make mutual concessions... Otherwise these talks are useless.

Kirill

#5 lore raw
In reply to Dave Hansen
Quoted 12 lines

kernel/fork.c static task_t *copy_process(unsigned lon · line 1138

Quoted 7 lines
kernel/fork.c renamed from kernel/fork.c.vpsinfo +1 -0
@@ -1132,6 +1133,7 @@ static task_t *copy_process(unsigned lon
       p->ioprio = current->ioprio;
 
       SET_LINKS(p);
+       (void)get_container(p->container);
       if (unlikely(p->ptrace & PT_PTRACED))
               __ptrace_link(p, current->parent); 

This entire patch looks nice and very straightforward, except for this bit. :) The "(void)" bit isn't usual kernel coding style. You can probably kill it.

it is to avoid warning message the value has no effect.

Quoted 4 lines

BTW, why does get_container() return the container argument? get_task_struct(), for instance is just a do{}while(0) loop, so it doesn't have a return value. Is there some magic later on in your patch set that utilizes this?

ok, I will remake it without a return value. not a real problem at all.

Quoted 4 lines

One other really minor thing: I usually try to do is keep the ! CONFIG_FOO functions static inlines, just like the full versions. The advantage is that you get some compile-time type checking, even when your CONFIG option is off.

it is not always appropriate :( I try to follow this as well :)

Kirill

#66 lore raw
In reply to Kirill Korotaev
Quoted 5 lines · Kirill Korotaev
Quoted 3 lines
Quoted 2 lines

The important properties of the proposed container implementation:
- each container has unique ID in the system

What namespace does this ID live in?

global namespace. can be virtualized later.
can be optional.

So a totally new kind of number in the kernel, that doesn't fall in any of the traditional namespaces. That is something I would like to avoid.

Quoted 2 lines

But the idea is simple. Eventually you will need some management tools anyway. And they should be able to refer to containers.

In my implementation I refer to things indirectly via the task
in it since I have an appropriate pid. As does the current
kernel code for the filesystem mount namespace.

So far except a tool to create my things I don't need special management tools the traditional unix ones suffice as I am simply extending the traditional unix model.

The best counter argument I have is that filesystem namespaces have already been implemented and we have not needed an additional id in the kernel.

Quoted 7 lines
Quoted 7 lines
Quoted 1 line

- each process in the kernel can belong to one container only

Reasonable.

Quoted 3 lines

- effective container pointer (econtainer()) is used on the task to avoid insertion of additional argument "container" to all functions where it is required.

Why is that desirable?

Ok. I can finally see the effective thing. Writing an argument onto the task structure instead of onto the stack seems weird to me.

Quoted 7 lines

It was discussed with Linus and the reason is provided in this text actually. There are 2 ways:
- to add additional argument "container" to all the functions where it is required.
Drawbacks:

a) lot's of changes,

I believe most of those changes make other places in the kernel where misuse happens easier to find. Turning places that need changes into compile errors seems a virtue to me. I do know there are places that store identifiers that might be used in a different context later on. Finding all of those places seems more important than generating compile errors for a few simple cases.

Quoted 1 line

b) compilation without virtualization is not the same.

Which is a reasonable consideration. But I expect most common kernels will ship with virtualization enabled so that is the important case to get correct.

Quoted 1 line

c) increased stakc usage

I think we are talking a few extra pointers on the stack. Our call nesting depth is not that great. So I would be surprised if the additional stack
usage would be significant. I expect the gains from the locality of
having everything in the same cache line are greater, than the gains from the reduced stack usage.

Quoted 4 lines

- to add effective container pointer on the task. i.e. context which kernel should be in when works with virtualized resources.
Drawbacks: a) there are some places where you need to change effective container context explicitly such as TCP/IP.

Yes.

Another is that it is easier to be aware and to think about what context you are working in if it is explicit. I expect other kernel developers are more likely to get it right if what is going is not hidden from them.

Quoted 5 lines
Quoted 4 lines
Quoted 2 lines

- kernel compilation with disabled virtualization should result in old good linux kernel

A reasonable goal.
Why do we need a container structure to hold pointers to other pointers?

can't catch what you mean :) is it prohibited somehow? :))))

No I just think an additional structure is unnecessary. Possibly it is a reasonable thing to do.

Quoted 8 lines
Quoted 2 lines

May I please be added to the CC list.
We are never going to form a consensus if all of the people doing

implementations don't

Quoted 1 line

talk.

To make a consensus people need to make mutual concessions... Otherwise these talks are useless.

Consensus does not require concessions. We are after technical excellence after all.

Yes people have to be willing to bend to work together to find the best solution. I am willing. At the same time I am not easy to convince that other solutions are necessarily better. To understand which is better we need to understand why each of us made the decisions we have made or hold the opinion we hold.

Once our reasons for doing things are clear and we can agree to a common set of goals, picking the best solution should not be too difficult.

My problem in previous talks is that I don't think you have seen where I have
been coming from. Which is why I shut up earlier and posted code.

Eric

#10 lore raw
In reply to Kirill Korotaev
Quoted 6 lines · Kirill Korotaev
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines

We are never going to form a consensus if all of the people doing implementations don't talk.

Speaking of which - it would be interesting to get Kirill's comments on Eric's patchset ;)

I'll do comment.

Thank you I will look forward to your comments.

Quoted 3 lines
Quoted 2 lines
Quoted 2 lines

Once we know what's good and bad about both patchsets, we'll be a lot closer to knowing what exactly should go upstream.

I'm starting to think that nothing in upstream can be better for all of us :)

In a thread voicing the concerns for maintaining out of tree patches that is a natural concern.

Quoted 11 lines
Quoted 9 lines

Let's compare approaches of patchsets before the patchsets themselves. It seems to be, should we:
A) make a general form of virtualising PIDs, and hope this assists later virtualisation efforts (Eric's patch)
B) make a general form of containers/jails/vservers/vpses, and layer PID virtualisation on top of it somewhere (as in openvz, vserver)

I can't think of any real use cases where you would specifically want A) without B).

Exactly! All these patches for A) look weird for me without containers itself. A try to make half-solution which is bad.

I am willing to contend that my approach also leads to a complete solution. In fact I believe my network virtualization has actually gone much farther
than yours. Although I admit there is still some work to do before
the code is in shape to be merged.

You notice in the kernel there is also not a struct process?

To me having a container structure while an obvious approach to the problem seems to add unnecessary policy to the kernel. Lumping together the implementation of multiple instances of different namespaces in a way that the implementation does not require.

Eric

#67 lore raw
In reply to Eric W. Biederman
Quoted 3 lines · Eric W. Biederman

Yes people have to be willing to bend to work together to find the best solution. I am willing. At the same time I am not easy to convince that other solutions are necessarily better.

It's not about "better", it's about different requirements.

Signature · 1 line
All Rights Reversed
#68 lore raw
Quoted 7 lines · Rik van Riel
Quoted 3 lines · Eric W. Biederman

Yes people have to be willing to bend to work together to find the best solution. I am willing. At the same time I am not easy to convince that other solutions are necessarily better.

It's not about "better", it's about different requirements.

In part but I think it is possible to largely agree on what the requirements for the kernel are, by seeing what the requirements of the different users are. That is part of the discussion.

Once those requirements are agreed upon it is about the best technical solution.

Maintainability is one of the more important requirements is it not?

Eric

#11 lore raw
In reply to Sam Vilain
Quoted 14 lines · Sam Vilain
Quoted 8 lines · Rik van Riel
Quoted 2 lines · Eric W. Biederman

We are never going to form a consensus if all of the people doing implementations don't talk.

Speaking of which - it would be interesting to get Kirill's comments on Eric's patchset ;)
Once we know what's good and bad about both patchsets, we'll be a lot closer to knowing what exactly should go upstream.

Let's compare approaches of patchsets before the patchsets themselves.

It seems to be, should we:

Quoted 8 lines

A) make a general form of virtualising PIDs, and hope this assists later virtualisation efforts (Eric's patch)

B) make a general form of containers/jails/vservers/vpses, and layer PID virtualisation on top of it somewhere (as in openvz, vserver)

I can't think of any real use cases where you would specifically want A) without B).

You misrepresent my approach.

First there is a huge commonality in the code bases between the different implementations and I have already gotten preliminary acceptance from the vserver developers, that my approach is sane. The major difference is what user interface does the kernel export, and I posted my user interface.

What user interface to export is a debate worth having.

For a lot of things getting the details just so is very important to long term maintainability and it is not my impression that anyone has done that yet.

Second I am not trying to just implement a form of virtualizing PIDs. Heck I don't intend to virtualize anything. The kernel has already virtualized everything I require. I want to implement multiple instances of the current kernel global namespaces. All I want is to be able to use the same name twice in user space and not have a conflict.

Beyond getting multiple instance of all of the kernel namespaces (which is the hard requirement for migration) my approach is to see what is needed for projects like vserver and vps and see how their needs can be met as well.

I disagree with a struct container simply because I do not see what value it happens to bring to the table. I have yet to see a problem that it solves that I have not solved yet.

In addition I depart from vserver and other implementations in another regard. It is my impression a lot of their work has been done so those projects are maintainable outside of the kernel, which makes sense as that is where those code bases live. But I don't think that gives the best solution for an in kernel implementation, which is what we are implementing.

So far I have succeeded in communicating with both the IBM and vserver developers. Hopefully I can do the same with Kirill Korotaev
and the OpenVz team. I think my implementation stands up to
criticism. But expect surprises in the way I solve a number of problems.

I suspect I will find similar surprises in the OpenVz code.

Time to do some more research I guess.

Eric

#12 lore raw
Quoted 4 lines
Quoted 2 lines

I can't think of any real use cases where you would specifically want A) without B).

You misrepresent my approach.

[...]

Quoted 6 lines

Second I am not trying to just implement a form of virtualizing PIDs. Heck I don't intend to virtualize anything. The kernel has already virtualized everything I require. I want to implement multiple instances of the current kernel global namespaces. All I want is to be able to use the same name twice in user space and not have a conflict.

if you want not virtualize anything, what is this discussion about? :)
can you provide an URL to your sources? you makes lot's of statements about that your network virtualization solution is better/more complete, so I'd like to see your solution in whole rather than only words. Probably this will help.

Quoted 3 lines

I disagree with a struct container simply because I do not see what value it happens to bring to the table. I have yet to see a problem that it solves that I have not solved yet.

again, source would help to understand your solution and problem you solved and not solved yet.

Quoted 6 lines

In addition I depart from vserver and other implementations in another regard. It is my impression a lot of their work has been done so those projects are maintainable outside of the kernel, which makes sense as that is where those code bases live. But I don't think that gives the best solution for an in kernel implementation, which is what we are implementing.

These soltuions are in kernel implementations actually.

Kirill

#15 lore raw
In reply to Sam Vilain
Quoted 28 lines · Sam Vilain
Quoted 11 lines · Rik van Riel
Quoted 1 line · Eric W. Biederman

We are never going to form a consensus if all of the people doing implementations don't talk.

Speaking of which - it would be interesting to get Kirill's comments on Eric's patchset ;)

Once we know what's good and bad about both patchsets, we'll be a lot closer to knowing what exactly should go upstream.

Let's compare approaches of patchsets before the patchsets themselves.

It seems to be, should we:

A) make a general form of virtualising PIDs, and hope this assists later virtualisation efforts (Eric's patch)

B) make a general form of containers/jails/vservers/vpses, and layer PID virtualisation on top of it somewhere (as in openvz, vserver)

I can't think of any real use cases where you would specifically want A) without B).

Quoted 9 lines

Also, the problem space in B) is now very well explored. To start with
A) would mean to throw away 4+ years of experience at this approach (just counting vserver and variants - not FreeBSD Jail, etc). Trying to re-base B) atop a massive refactoring and new patch like A) would incur a lot of work; however fitting it into B) is natural and solved conceptually and in practice, with the only drawback I see being that the use cases mentioned above wouldn't suffer from the side-effects of B).

Sam, that is a bit far fetched. I looked and experienced myself with both approaches and there is a lot of functional overlap, with both of them having advantages and disadvantages.
What Eric provides is an alternative to the PID virtualization part of openvz. Indeed it is a pid isolation more then anything else (with some dealing at the boundary condition).
I personally don't see much problem in replacing the pid virtualization of openvz with that of pidspaces.
So the correct thing to do here is as RvR points out, simple discuss the merits and drawbacks of each PID approach for now and settle on one and move on....

Here are my two cents on this.

The pid-namespace (pspace) provides an approach of fully separate the allocation and maintenance of the pids and treating the <pspace,pid> tuple as an entity to uniquely identify a task and vice versa.
As a result the logic of lookup can be pushed down the find_task_by_pid() lookup. There are specific cases where the init_task of a container or pspace needs to be checked to ensure that signals/waits and alike are properly handled across pspace boundaries. I think this is an intuitive and clean way. It also completely avoids the problem of having to think about all the locations at the user/kernel boundary where a vpid/pid conversion needs to take place. It also avoids the problems that logically vpids and pids are different types and therefore it would have been good to have type checking automatically identify problem spots.
On the negative side, it does require to maintain a pidmap per pidspace.

The vpid approach has the drawbacks of having to identify the conversion spots of all vpid vs. pid semantics. On the otherhand it does take advantage of the fact that no virtualization has to take place until a "container" has been migrated, thus rendering most of the vpid<->pid calls to be noops.

What I like about the pspace approach is that it explicitely defines in the code when I am using a different pspace for the lookup. That is kind of hidden in the vpid/pid approach.

The container is just an umbrella object that ties every "virtualized" subsystem together.

So, what do other folks, see as pluses and minus of each approach. Once we have a more complete listing of these, maybe the decision becomes more obvious !

Regards
-- Hubertus

#13 lore raw
In reply to Kirill Korotaev
Quoted 12 lines · Kirill Korotaev
Quoted 4 lines
Quoted 2 lines

I can't think of any real use cases where you would specifically want A) without B).

You misrepresent my approach.

[...]

Quoted 6 lines

Second I am not trying to just implement a form of virtualizing PIDs. Heck I don't intend to virtualize anything. The kernel has already virtualized everything I require. I want to implement multiple instances of the current kernel global namespaces. All I want is to be able to use the same name twice in user space and not have a conflict.

Quoted 5 lines

if you want not virtualize anything, what is this discussion about? :) can you provide an URL to your sources? you makes lot's of statements about that your network virtualization solution is better/more complete, so I'd like to see your solution in whole rather than only words.
Probably this will help.

Sure.

I think it is more an accident of time, and the fact that I am quite proud of where I am at. You quite likely have improved things in openvz since last I looked as well. Currently my code quality is only a proof of concept but the tree is below.

git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm/linux-2.6-ns.git/

Basically the implementation appears to the user as a separate instance of the network stack. Since the tree was experimental the path is absolutely horrible. I am in the process of cleaning and redoing all of that now in preparation for kernel inclusion.

But I suspect I am in the lead as no one else had noticed the ipv6 reference counting bugs.

Quoted 5 lines
Quoted 3 lines

I disagree with a struct container simply because I do not see what value it happens to bring to the table. I have yet to see a problem that it solves that I have not solved yet.

again, source would help to understand your solution and problem you solved and not solved yet.

Above. But at least with pids it has all been posted on the mailing list.

I think I have solved most of the code structural issues and the big kernel API issues. A lot of the little things I have not gotten to yet as I figured it was best approached later.

Quoted 7 lines
Quoted 6 lines

In addition I depart from vserver and other implementations in another regard. It is my impression a lot of their work has been done so those projects are maintainable outside of the kernel, which makes sense as that is where those code bases live. But I don't think that gives the best solution for an in kernel implementation, which is what we are implementing.

These soltuions are in kernel implementations actually.

Sorry in/out in this context I was referring to the stock linux kernel. As soon as I had a viable proof of concept I began working to get my code merged.

Eric

#16 lore raw
In reply to Hubertus Franke
Quoted 11 lines · Hubertus Franke

The vpid approach has the drawbacks of having to identify the conversion spots
of all vpid vs. pid semantics. On the otherhand it does take advantage of the fact that no virtualization has to take place until a "container" has been migrated, thus rendering most of the vpid<->pid calls to be noops.

What I like about the pspace approach is that it explicitely defines in the code
when I am using a different pspace for the lookup. That is kind of hidden in the vpid/pid approach.

I agree with this. From a maintenance pov, imagining making a minor change to some pid-related code, if I see something doing effectively "if (pspace1 == pspace2 && pid1==pid2)" that is clear, whereas trying to remember whether I'm supposed to return the pid or vpid can get really confusing. We actually had some errors with that while we were developing the first vpid patchset we posted in december.

I believe that from a vserver point of view, either approach will work. You either create a new pspace and make 'init' pid 1 in that pspace, or, in the openvz approach, you start virtualizing with a hashtable so userspace in the new vserver/container/vz/whateveritscalled sees that init as pid1, while the rest of the system sees it as pid 3270 or something.

Likewise, for checkpoint/restore and migration, either approach works. All we really need is, on restore/migrate, to be able to create processes with their original pids, so we can do that either with real pids in a new container, or virtualized pids faked for process in the same vz.

Are there other uses of pid virtualization which one approach or the other cannot accomodate?

If not, then I for one lean towards the more maintainable code. (which I'm sure we're not agreed on is Eric's, but imvho it is)

-serge

#17 lore raw
Quoted 27 lines · Serge E. Hallyn
Quoted 9 lines · Hubertus Franke

The vpid approach has the drawbacks of having to identify the conversion spots of all vpid vs. pid semantics. On the otherhand it does take advantage of the fact that no virtualization has to take place until a "container" has been migrated, thus rendering most of the vpid<->pid calls to be noops.

What I like about the pspace approach is that it explicitely defines in the code when I am using a different pspace for the lookup. That is kind of hidden in the vpid/pid approach.

I agree with this. From a maintenance pov, imagining making a minor change to some pid-related code, if I see something doing effectively "if (pspace1 == pspace2 && pid1==pid2)" that is clear, whereas trying to remember whether I'm supposed to return the pid or vpid can get really confusing. We actually had some errors with that while we were developing the first vpid patchset we posted in december.

I believe that from a vserver point of view, either approach will work. You either create a new pspace and make 'init' pid 1 in that pspace, or, in the openvz approach, you start virtualizing with a hashtable so userspace in the new vserver/container/vz/whateveritscalled sees that init as pid1, while the rest of the system sees it as pid 3270 or something.

Quoted 8 lines

Likewise, for checkpoint/restore and migration, either approach works. All we really need is, on restore/migrate, to be able to create processes with their original pids, so we can do that either with real pids in a new container, or virtualized pids faked for process in the same vz.

Are there other uses of pid virtualization which one approach or the other cannot accomodate?

Kirill brought up that VPS can span a cluster..
if so how do you (Kirill) do that? You pre-partition the pids into allocation ranges for each container?
Eitherway, if this is an important feature, then one needs to look at how that is achieved in pspace (e.g. mod the pidmap_alloc() function to take legal ranges into account). Should still be straight forward.

Quoted 6 lines

If not, then I for one lean towards the more maintainable code. (which I'm sure we're not agreed on is Eric's, but imvho it is)

-serge

#18 lore raw
Quoted 6 lines · Hubertus Franke

Kirill brought up that VPS can span a cluster..
if so how do you (Kirill) do that? You pre-partition the pids into allocation ranges for each container?
Eitherway, if this is an important feature, then one needs to look at how that is achieved in pspace (e.g. mod the pidmap_alloc() function to take legal ranges into account). Should still be straight forward.

Actually legal ranges already exist in the form of min/max values. So that is trivial to implement.

Eric

#20 lore raw

The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke

I think I can boil the discussion down into some of the fundamental questions that we are facing.

Currently everyone seems to agree that we need something like my namespace concept that isolates multiple resources.

We need these for
PIDS
UIDS
SYSVIPC
NETWORK
UTSNAME
FILESYSTEM
etc.

The questions seem to break down into:
1) Where do we put the references to the different namespaces?
- Do we put the references in a struct container that we reference from struct task_struct?
- Do we put the references directly in struct task_struct?

2) What is the syscall interface to create these namespaces?
- Do we add clone flags?
(Plan 9 style)
- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?
- Do we in addition add syscalls to manipulate containers generically?

I don't think having a single system call to create a container and a new instance of each namespace is reasonable as that does not give us a path into the future when we create yet another namespace.

If we have one syscall per each namespace why would we need a container structure?

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?
- Do we assign some kind of unique id to the containers?

4) How do we implement each of these namespaces? Besides being maintainable are there other constraints?

5) How do we control the resource inside a namespace starting from a process that is outside of that namespace?
- The filesystem mount namespace gave an interesting answer. So it is quite possible other namespaces will give equally interesting and surprising answers.

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

7) How do we allow a process inside a container to create containers for it's children?
- In general this is trivial but there are a few ugly issues here.

I think these are the key questions of the conversation.

Personally so long as we get true namespaces, implemented in a performant and maintainable way that a process from the inside can't distinguish from what we have now I have no hard requirements.

Eric

#19 lore raw
In reply to Eric W. Biederman
Quoted 12 lines · Eric W. Biederman
Quoted 6 lines · Hubertus Franke

Kirill brought up that VPS can span a cluster..
if so how do you (Kirill) do that? You pre-partition the pids into allocation ranges for each container?
Eitherway, if this is an important feature, then one needs to look at how that is achieved in pspace (e.g. mod the pidmap_alloc() function to take legal ranges into account). Should still be straight forward.

Quoted 3 lines

Actually legal ranges already exist in the form of min/max values. So that is trivial to implement.

Yipp, didn't want to state the obvious, but also give Kirrill a chance to explain how its done in OpenVZ.

Ultimately, the same "partitioning" that works on vps_info, should work on pspace.

-- Hubertus

#14 lore raw
In reply to Eric W. Biederman

Eric W. Biederman wrote [note: quoting sections out of order]:

Quoted 8 lines
Quoted 6 lines · Sam Vilain

Let's compare approaches of patchsets before the patchsets themselves. It seems to be, should we:
A) make a general form of virtualising PIDs, and hope this assists later virtualisation efforts (Eric's patch)
I can't think of any real use cases where you would specifically want A) without B).

You misrepresent my approach.

ok, after reading more of your post, agreed.

Quoted 1 line

What user interface to export is a debate worth having.

This is the bit that needs a long period of prototyping and experimental use IMHO. So in essence, we're agreeing on that point.

Quoted 11 lines

First there is a huge commonality in the code bases between the different implementations and I have already gotten preliminary acceptance from the vserver developers, that my approach is sane. The major difference is what user interface does the kernel export, and I posted my user interface.
Second I am not trying to just implement a form of virtualizing PIDs. Heck I don't intend to virtualize anything. The kernel has already virtualized everything I require. I want to implement multiple instances of the current kernel global namespaces. All I want is to be able to use the same name twice in user space and not have a conflict.

Right, well, I think our approaches might have more in common than I previously thought.

Indeed, it seems that at least one of the features of Linux-VServer I am preparing for consideration for inclusion into Linus' tree are your work :-).

Quoted 4 lines

Beyond getting multiple instance of all of the kernel namespaces (which is the hard requirement for migration) my approach is to see what is needed for projects like vserver and vps and see how their needs can be met as well.

ok, but the question is - doesn't this just constitute a refactoring once the stable virtualisation code is in?

I'm just a bit nervous about trying to refactor-approach-and-concepts-as-we-go.

But look, I'll take a closer look at your patches, and see if I can merge with you anyhow. Thanks for the git repo!

Sam.

#52 lore raw
In reply to Hubertus Franke
Quoted 1 line · Hubertus Franke

The container is just an umbrella object that ties every "virtualized" subsystem together.

I like this description; it matches roughly with the concepts as presented by vserver; there is the process virtualisation (vx_info), and the network virtualisation (nx_info) of Eric's that has been integrated to the vserver 2.1.x development branch. However the vx_info has become the de facto umbrella object space as well. These could almost certainly be split out without too much pain or incurring major rethinks.

Sam.

#53 lore raw
Quoted 13 lines · Sam Vilain
Quoted 1 line · Hubertus Franke

The container is just an umbrella object that ties every "virtualized" subsystem together.

I like this description; it matches roughly with the concepts as presented by vserver; there is the process virtualisation (vx_info), and the network virtualisation (nx_info) of Eric's that has been integrated to the vserver 2.1.x development branch. However the vx_info has become the de facto umbrella object space as well. These could almost certainly be split out without too much pain or incurring major rethinks.

Quoted 2 lines

Sam.

Agreed.. here are some issued we learned from other projects that had similar interception points.

Having a central umbrella object (let's stick to the name container) is useful, but being the only object through which every access has to pass may have drawbacks..

task->container->pspace->pidmap[offset].page implies potential
cachemisses etc.

If overhead becomes too large, then we can stick (cache) the pointer additionally in the task struct. But ofcourse that should be carefully examined on a per subsystem base...

==
Another thing to point out is that container's can have overlaps.

C/R should be a policy thing. So if each "subsystem"

Quoted 7 lines

Quote Eric>>>
PIDS
UIDS
SYSVIPC
NETWORK
UTSNAME
FILESYSTEM

is represented as a NAMESPACE, then one can pick and choose as a policy how these constitute at a conceptual level as a container. You want something migratable you better make sure that container implies unique subsystems.
Maybe you want to nest containers, but only want to create a separate pidspaces for performance isolation (see planetlab work with vserver).
So, there are many possibilities, that might make perfect sense for different desired solutions and it seems with the
clone ( CLONE_FLAGS_NSPACE_[PIDS/UIDS/SYS.../FS] ) one gets a solution that is flexible, yet embodies may requirements.....

-- Hubertus

#21 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Eric W. Biederman
Quoted 3 lines · Eric W. Biederman

I think I can boil the discussion down into some of the fundamental questions that we are facing.

Man, bearly can keep up with this email load. Addressed some in previous thread, but will reiterate under this context.

Quoted 11 lines

Currently everyone seems to agree that we need something like my namespace concept that isolates multiple resources.

We need these for PIDS
UIDS
SYSVIPC
NETWORK
UTSNAME
FILESYSTEM
etc.

Quoted 4 lines

The questions seem to break down into:
1) Where do we put the references to the different namespaces?
- Do we put the references in a struct container that we reference from struct task_struct?
- Do we put the references directly in struct task_struct?

You "cache" task_struct->container->hotsubsys under task_struct->hotsubsys.
We don't change containers other then at clone time, so no coherency issue here !!!! Which subsystems pointers to "cache", should be agreed by the experts, but first approach should always not to cache and go through the container.

Quoted 4 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags?
(Plan 9 style)

Like that approach .. flexible .. particular when one has well specified namespaces.

Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

Where does that approach end .. what's wrong with doing it at clone() time ? Mainly the naming issue. Just providing a flag does not give me name.

Quoted 6 lines

- Do we in addition add syscalls to manipulate containers generically?

I don't think having a single system call to create a container and a new instance of each namespace is reasonable as that does not give us a path into the future when we create yet another namespace.

Agreed.

Quoted 7 lines

If we have one syscall per each namespace why would we need a container structure?

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?
- Do we assign some kind of unique id to the containers?

In containers I simply created an explicite name, which ofcourse colides with the clone() approach ..
One possibility is to allow associating a name with a namespace.
For instance
int set_namespace_name( long flags, const char *name ) /* the once we are using in clone */ {
if (!flag)
set name of container associated with current.
if (flag())
set the name if only one container is associated with the namespace(s) identified .. or some similar rule
}

Quoted 5 lines

4) How do we implement each of these namespaces? Besides being maintainable are there other constraints?

Good question... at least with PID and FS two are there ..

Quoted 8 lines

5) How do we control the resource inside a namespace starting from a process that is outside of that namespace?
- The filesystem mount namespace gave an interesting answer. So it is quite possible other namespaces will give equally interesting and surprising answers.

Quoted 5 lines

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

Possible to split the lock up now.. one for each pidspace ?

Quoted 5 lines

7) How do we allow a process inside a container to create containers for it's children?
- In general this is trivial but there are a few ugly issues here.

Speaking of pids only here ...
Does it matter, you just hang all those containers hang of init. What ever hierarchy they form is external ...

Quoted 11 lines

I think these are the key questions of the conversation.

Personally so long as we get true namespaces, implemented in a performant and maintainable way that a process from the inside can't distinguish from what we have now I have no hard requirements.

Eric

-- Hubertus

#22 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Hello!

Quoted 11 lines
Quoted 3 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags?
(Plan 9 style)

Like that approach .. flexible .. particular when one has well specified namespaces.

Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

Where does that approach end .. what's wrong with doing it at clone() time ?

That most of those namespaces need a special setup rather than a plain copy?

F.e. what are you going to do with NETWORK namespace? The only valid thing to do is to prepare a new context and to configure its content (addresses, routing tables, iptables...) later. So that, in this case it is natural to inherit the context through clone() and to create new context with a separate syscall.

Seems, only PID space needs to be setup at clone time. All the rest of suggested namespaces are more convenient to change with separate syscalls.

I would suggest to combine both approaches. Those namespaces, which can be naturally copied while clone() (f.e. the best example is already existing CLONE_NEWNS) deserve a clone() flag. The rest are preserved through clone() and forked and configured later.

Alexey

#58 lore raw
In reply to Sam Vilain
Quoted 12 lines · Sam Vilain
Quoted 2 lines · Hubertus Franke

The container is just an umbrella object that ties every "virtualized" subsystem together.

I like this description; it matches roughly with the concepts as presented by vserver; there is the process virtualisation (vx_info), and the network virtualisation (nx_info) of Eric's that has been integrated to the vserver 2.1.x development branch. However the vx_info has become the de facto umbrella object space as well. These could almost certainly be split out without too much pain or incurring major rethinks.

Quoted 1 line

Sam.

How does all of this tie in with CPU Sets? It seems to me, they have something not unlike a container already that supports nesting.

Kevin

Quoted 5 lines

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#59 lore raw
Quoted 10 lines · Kevin Fox
Quoted 8 lines
Quoted 1 line

The container is just an umbrella object that ties every "virtualized" subsystem together.

I like this description; it matches roughly with the concepts as presented by vserver; there is the process virtualisation (vx_info), and the network virtualisation (nx_info) of Eric's that has been integrated to the vserver 2.1.x development branch. However the vx_info has become the de facto umbrella object space as well. These could almost certainly be split out without too much pain or incurring major rethinks.

How does all of this tie in with CPU Sets? It seems to me, they have something not unlike a container already that supports nesting.

Yes, I saw that. AIUI that's mainly about binding groups of processes to CPUs, to defeat Amdahl's law when it rears its head. It fits into the containers model as a hard partitioning feature, but is a lot more crude than the CPU Token Bucket scheduler in Linux-VServer.

No doubt both CPU allocation strategies will be useful.

Sam.

#23 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Alexey Kuznetsov
Quoted 14 lines · Alexey Kuznetsov

Hello!

Quoted 11 lines
Quoted 3 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags?
(Plan 9 style)

Like that approach .. flexible .. particular when one has well specified namespaces.

Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

Where does that approach end .. what's wrong with doing it at clone() time ?

Quoted 7 lines

That most of those namespaces need a special setup rather than a plain copy?

F.e. what are you going to do with NETWORK namespace? The only valid thing to do is to prepare a new context and to configure its content (addresses, routing tables, iptables...) later. So that, in this case it is natural to inherit the context through clone() and to create new context with a separate syscall.

With a NETWORK namespace what I implemented was that you get a empty namespace with a loopback interface.

But setting up the namespace from the inside is clearly the sane thing todo.

Quoted 2 lines

Seems, only PID space needs to be setup at clone time. All the rest of suggested namespaces are more convenient to change with separate syscalls.

Actually I think I can setup a PID space in a syscall as well. It is a little odd that your session, and process group change but since I was keeping 2 pids on the PID space leader I could easily do it. The fact that getpid() would start returning 1 might be confusing to a some processes though so clone seems to be the natural time to do it.

Quoted 4 lines

I would suggest to combine both approaches. Those namespaces, which can be naturally copied while clone() (f.e. the best example is already existing CLONE_NEWNS) deserve a clone() flag. The rest are preserved through clone() and forked and configured later.

Sounds reasonable. We make the decision on a case by case base whatever make sense for the patch and the namespace.

The only real advantage to clone is you can create a bunch of namespaces all in one shot. Of course that makes sys_clone a little slower.

Eric

#24 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 24 lines · Eric W. Biederman
Quoted 14 lines · Alexey Kuznetsov

Hello!

Quoted 11 lines
Quoted 3 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags?
(Plan 9 style)

Like that approach .. flexible .. particular when one has well specified namespaces.

Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

Where does that approach end .. what's wrong with doing it at clone() time ?

Quoted 7 lines

That most of those namespaces need a special setup rather than a plain copy?

F.e. what are you going to do with NETWORK namespace? The only valid thing to do is to prepare a new context and to configure its content (addresses, routing tables, iptables...) later. So that, in this case it is natural to inherit the context through clone() and to create new context with a separate syscall.

Quoted 5 lines

With a NETWORK namespace what I implemented was that you get a empty namespace with a loopback interface.

But setting up the namespace from the inside is clearly the sane thing todo.

What I tried to do in a proof of concept long ago was to have CLONE_NETNS mean that you get access to all the network devices, but then you could drop/add them. Conceptually I prefer that to getting an empty namespace, but I'm not sure whether there's any practical use where you'd want that...

-serge

#25 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 6 lines · Serge E. Hallyn

What I tried to do in a proof of concept long ago was to have CLONE_NETNS mean that you get access to all the network devices, but then you could drop/add them. Conceptually I prefer that to getting an empty namespace, but I'm not sure whether there's any practical use where you'd want that...

My observation was that the network stack does not come out cleanly as a namespace unless you adopt the rule that a network device belongs to exactly one network namespace.

With that rule dealing with the network stack is just a matter of making some currently global variables/data structures per container.

A pain to do the first round but easy to maintain once you are there and the logic of the code doesn't need to change.

Eric

#60 lore raw
In reply to Sam Vilain

The driving force for cpusets are NUMA architectures.

Cpusets represent the topologies of NUMA systems, with hierarchies of cabinets, drawers, boards, packages, cores, hyperthreads, and with chunks of main memory associated usually with the board, but sometimes a layer or two up or down.

Since not all cpus have the same access performance (delay and bandwidth) to all memory chunks (nodes), for optimum performance one wants to bind tasks, cpus and memory together, so as to run tasks on sets of cpus and memory that are "near" to each other, and to size the sets appropriately for the workload presented by the tasks.

Cpusets have no explicit awareness of topology; they just provides a file system style hierarchy of named, permissioned sets, where each set has:
mems - the memory nodes in that set
cpus - the cpus in that set
tasks - the tasks running on these cpus and mems

For any cpuset, the 'cpus' and 'mems' are a subset of its parent in the hierarchy, and the root of the hierarchy (usually mounted at /dev/cpuset) contains all the online cpus and mems in the system.

Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#26 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Eric W. Biederman
Quoted 9 lines · Eric W. Biederman
Quoted 6 lines · Serge E. Hallyn

What I tried to do in a proof of concept long ago was to have CLONE_NETNS mean that you get access to all the network devices, but then you could drop/add them. Conceptually I prefer that to getting an empty namespace, but I'm not sure whether there's any practical use where you'd want that...

Quoted 3 lines

My observation was that the network stack does not come out cleanly as a namespace unless you adopt the rule that a network device belongs to exactly one network namespace.

yep, that's what the first network virtualization for Linux-VServer aimed at, but found too complicated the second one uses 'pairs' of communicating devices to send between guests/host

Quoted 2 lines

With that rule dealing with the network stack is just a matter of making some currently global variables/data structures per container.

yep, like the universal loopback and so ...

Quoted 2 lines

A pain to do the first round but easy to maintain once you are there and the logic of the code doesn't need to change.

best,
Herbert

Quoted 1 line

Eric

#27 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 4 lines · Herbert Poetzl

yep, that's what the first network virtualization for Linux-VServer aimed at, but found too complicated the second one uses 'pairs' of communicating devices to send between guests/host

Well you need the pairs of course, to communication between the two stack ``instances''.

Quoted 4 lines
Quoted 2 lines

With that rule dealing with the network stack is just a matter of making some currently global variables/data structures per container.

yep, like the universal loopback and so ...

:)

Eric

#46 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Eric W. Biederman
Quoted 16 lines · Eric W. Biederman

I think I can boil the discussion down into some of the fundamental questions that we are facing.

Currently everyone seems to agree that we need something like my namespace concept that isolates multiple resources.

We need these for
PIDS
UIDS
SYSVIPC
NETWORK
UTSNAME
FILESYSTEM
etc.

Quoted 4 lines

The questions seem to break down into:
1) Where do we put the references to the different namespaces?
- Do we put the references in a struct container that we reference from struct task_struct?

no, just let the tasks be in groups of disjunct spaces so that they can have shared or private structures for each of the identified spaces

Quoted 1 line

- Do we put the references directly in struct task_struct?

yes, IMHO that's the way to do it .. Linux-VServer is moving in this direction for some time now, but we need to add a special space for context capabilities and context flags, basically a context struct, similar to a namespace ...

Quoted 3 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags?
(Plan 9 style)

I'd definitely prefer that, maybe if necessary with a 'new' clone syscall which allows to do a little more than clone does now, e.g.

- clone into container/context/guest
- set space initializations and limits, etc ...

Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

doesn't make sense for the creation, but a syscall for moving between and management of spaces are very important ...

Quoted 10 lines

- Do we in addition add syscalls to manipulate containers generically?

I don't think having a single system call to create a container and a new instance of each namespace is reasonable as that does not give us a path into the future when we create yet another namespace.

If we have one syscall per each namespace why would we need a container structure?

for the beforementioned permissions and flags, but you can as well see it as separate 'context' space

Quoted 3 lines

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?

the process will be an unique identifier to the namespace, but it might not be easy to use it, so IMHO it might at least make sense to ...

Quoted 1 line

- Do we assign some kind of unique id to the containers?

have an unique identifier for the context space so that somebody can cherry pick namespaces from there

(in this case, the context space would hold references to the other namespaces which are the ones used by new tasks created into a context, basically a template for them)

Quoted 2 lines

4) How do we implement each of these namespaces? Besides being maintainable are there other constraints?

extensible and with keeping hierarchical structures in mind .. would be bad if we could not do sub-contexts without a complete rewrite

Quoted 2 lines

5) How do we control the resource inside a namespace starting from a process that is outside of that namespace?

depends on the resource, but limits have to go to the context structure, so that they apply for the entire context space, not just for a task

Quoted 10 lines

- The filesystem mount namespace gave an interesting answer. So it is quite possible other namespaces will give equally interesting and surprising answers.

6) How do we do all of this efficiently without a noticeable impact on performance?

- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

well, we have to be careful with the complexity ... keep things like refcounting and 'magic' on clone simple, for the default case ...

Quoted 5 lines

7) How do we allow a process inside a container to create containers for it's children?

- In general this is trivial but there are a few ugly issues here.

a flat implementation will work for a hierarchical design if certain things are handled properly, just think resource management for sub-contexts and changes in the parent's limits ...

Quoted 6 lines

I think these are the key questions of the conversation.

Personally so long as we get true namespaces, implemented in a performant and maintainable way that a process from the inside can't distinguish from what we have now I have no hard requirements.

keep up the good work!

best,
Herbert

Quoted 1 line

Eric

#54 lore raw
In reply to Hubertus Franke
Quoted 8 lines · Hubertus Franke

Agreed.. here are some issued we learned from other projects that had similar interception points.

Having a central umbrella object (let's stick to the name container) is useful, but being the only object through which every access has to pass may have drawbacks..

Quoted 6 lines

task->container->pspace->pidmap[offset].page implies potential
cachemisses etc.

If overhead becomes too large, then we can stick (cache) the pointer additionally in the task struct. But ofcourse that should be carefully examined on a per subsystem base...

Ok. After talking with the vserver guys on IRC. I think grasp the importance. The key feature is to have a place to put limits and the like for your entire container. Look at all of the non-signal stuff in struct signal for an example. The nested namespaces seem to be just an implementation detail.

For OpenVZ having the other namespaces nested may have some importance. I haven't gotten their yet.

The task->container->pspace->.... thing feels very awkward to me, and feels like it increases our chance getting a cache miss.

So I support the concept of a place to put all of the odd little things like rlimits for containers. But I would like to flatten it in the task_struct if we can.

Eric

#29 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke
Quoted 6 lines · Hubertus Franke
Quoted 3 lines · Eric W. Biederman

I think I can boil the discussion down into some of the fundamental questions that we are facing.

Man, bearly can keep up with this email load. Addressed some in previous thread, but will reiterate under this context.

:)

Quoted 24 lines
Quoted 4 lines

Currently everyone seems to agree that we need something like my namespace concept that isolates multiple resources. We need these for PIDS
UIDS

Quoted 8 lines

SYSVIPC
NETWORK
UTSNAME
FILESYSTEM
etc.
The questions seem to break down into:
1) Where do we put the references to the different namespaces?
- Do we put the references in a struct container that we reference from struct

task_struct?

Quoted 1 line

- Do we put the references directly in struct task_struct?

You "cache" task_struct->container->hotsubsys under task_struct->hotsubsys.
We don't change containers other then at clone time, so no coherency issue here !!!!
Which subsystems pointers to "cache", should be agreed by the experts, but first approach should always not to cache and go through the container.

Quoted 2 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags? (Plan 9 style)

Quoted 8 lines

Like that approach .. flexible .. particular when one has well specified namespaces.

Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

Where does that approach end .. what's wrong with doing it at clone() time ? Mainly the naming issue. Just providing a flag does not give me name.

It really is a fairly even toss up. The usual argument for doing it this way is that you will get a endless stream of arguments added to fork+exec other wise. Look of posix_spawn or the windows version if you want an example. Bits to clone are skirting the edge of a slippery slope.

Quoted 12 lines
Quoted 4 lines

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?
- Do we assign some kind of unique id to the containers?

In containers I simply created an explicite name, which ofcourse colides with the
clone() approach ..
One possibility is to allow associating a name with a namespace. For instance
int set_namespace_name( long flags, const char *name ) /* the once we are using in clone */

Quoted 9 lines
{
	if (!flag)
		set name of container associated with current.
	if (flag())
		set the name if only one container is associated with the

namespace(s)
identified .. or some similar rule
}

What I have done which seems easier than creating new names is to refer to the process which has the namespace I want to manipulate.

Quoted 7 lines
Quoted 5 lines

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

Possible to split the lock up now.. one for each pidspace ?

At the moment it is worth thinking about. If the problem isn't so bad that people aren't actively working on it we don't have to solve the problem for a little while, just be aware of it.

Quoted 8 lines
Quoted 4 lines

7) How do we allow a process inside a container to create containers for it's children?
- In general this is trivial but there are a few ugly issues here.

Speaking of pids only here ...
Does it matter, you just hang all those containers hang of init. What ever hierarchy they form is external ...

In general it is simple. For resource accounting, and for naming so you can migrate a container with a nested container it is a question you need to be slightly careful with.

Eric

#55 lore raw
In reply to Eric W. Biederman
Quoted 26 lines · Eric W. Biederman
Quoted 7 lines · Hubertus Franke

Agreed.. here are some issued we learned from other projects that had similar interception points.

Having a central umbrella object (let's stick to the name container) is useful, but being the only object through which every access has to pass may have drawbacks..

Quoted 6 lines

task->container->pspace->pidmap[offset].page implies potential
cachemisses etc.

If overhead becomes too large, then we can stick (cache) the pointer additionally in the task struct. But ofcourse that should be carefully examined on a per subsystem base...

Ok. After talking with the vserver guys on IRC. I think grasp the importance. The key feature is to have a place to put limits and the like for your entire container. Look at all of the non-signal stuff in struct signal for an example. The nested namespaces seem to be just an implementation detail.

For OpenVZ having the other namespaces nested may have some importance. I haven't gotten their yet.

Quoted 7 lines

The task->container->pspace->.... thing feels very awkward to me, and feels like it increases our chance getting a cache miss.

So I support the concept of a place to put all of the odd little things like rlimits for containers. But I would like to flatten it in the task_struct if we can.

My point was to mainly identify the performance culprits and provide an direct access to those "namespaces" for performance reasons. So we all agreed on that we need to do that..

Question now (see other's note as well), should we provide a pointer to each and every namespace in struct task. Seem rather wasteful to me as certain path/namespaces are not exercise heavily.

Having one object "struct container" that still embodies all namespace still seems a reasonable idea.
Otherwise identifying the respective namespace of subsystems will have to go through container->init->subsys_namespace or similar. Not necessarily bad either..

-- Hubertus

#47 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Herbert Poetzl
Quoted 7 lines · Herbert Poetzl
Quoted 3 lines

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?

the process will be an unique identifier to the
namespace, but it might not be easy to use it, so
IMHO it might at least make sense to ...

Especially from userspace. If I want to start a checkpoint on a container, but I have to use the process to identify the container/namespace, well I can't uniquely specify the process by pid anymore...

-serge

#30 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Eric W. Biederman
Quoted 16 lines · Eric W. Biederman
Quoted 13 lines · Hubertus Franke
Quoted 2 lines · Eric W. Biederman

2) What is the syscall interface to create these namespaces?
- Do we add clone flags? (Plan 9 style)

Like that approach .. flexible .. particular when one has well specified namespaces.

Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

Quoted 9 lines
Quoted 2 lines

Where does that approach end .. what's wrong with doing it at clone() time ? Mainly the naming issue. Just providing a flag does not give me name.

It really is a fairly even toss up. The usual argument for doing it this way is that you will get a endless stream of arguments added to fork+exec other wise. Look of posix_spawn or the windows version if you want an example. Bits to clone are skirting the edge of a slippery slope.

So it seems the clone( flags ) is a reasonable approach to create new namespaces. Question is what is the initial state of each namespace? In pidspace we know we should be creating an empty pidmap !
In network, someone suggested creating a loopback device
In uts, create "localhost"
Are there examples where we rather inherit ? Filesystem ?
Can we iterate the assumption for each subsystem what people thing is right?

IMHO, there is only a need to refer to a namespace from the global context. Since one will be moving into a new container, but getting out of one could be prohibitive (e.g. after migration)
It does not make sense therefore to know the name of a namespace in a different container.

The example you used below by using the pid comes natural, because that already limits visibility.

I am still struggling with why we need new sys_calls.
sys_calls already exist for changing certain system parameters (e.g. utsname ) so to me it boils down to identifying a proper initial state when the namespace is created.

Quoted 24 lines
Quoted 13 lines
Quoted 5 lines

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?
- Do we assign some kind of unique id to the containers?

In containers I simply created an explicite name, which ofcourse colides with the
clone() approach ..
One possibility is to allow associating a name with a namespace. For instance
int set_namespace_name( long flags, const char *name ) /* the once we are using in clone */

Quoted 9 lines
{
	if (!flag)
		set name of container associated with current.
	if (flag())
		set the name if only one container is associated with the

namespace(s)
identified .. or some similar rule
}

Quoted 2 lines

What I have done which seems easier than creating new names is to refer to the process which has the namespace I want to manipulate.

Is then the idea to only allow the container->init to manipulate or is there need to allow other priviliged processes to perform namespace manipulation?
Also after thinking about it.. why is there a need to have an external name for a namespace ?

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

Quoted 7 lines
Quoted 1 line

Possible to split the lock up now.. one for each pidspace ?

At the moment it is worth thinking about. If the problem isn't so bad that people aren't actively working on it we don't have to solve the problem for a little while, just be aware of it.

Agree, just need to be sure we can split it up. But you already keep a task list per pid-namespace, so there should be no problem IMHO. If so let's do it now and take it of the table it its as simple as

task_list_lock ::= pspace->task_list_lock

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

7) How do we allow a process inside a container to create containers for it's children?
- In general this is trivial but there are a few ugly issues here.

Quoted 8 lines
Quoted 3 lines

Speaking of pids only here ...
Does it matter, you just hang all those containers hang of init. What ever hierarchy they form is external ...

In general it is simple. For resource accounting, and for naming so you can migrate a container with a nested container it is a question you need to be slightly careful with.

Absolutely, that's why it is useful to have an "external" idea of how containers are constructed of basic namespaces==subsystems. The it "simply" becomes a policy. E.g. one can not migrate a container that has shared subsystems.
Resource accounting I agree, that might required active aggregation at request time.

-- Hubertus

#48 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Serge E. Hallyn
Quoted 10 lines · Serge E. Hallyn
Quoted 6 lines · Herbert Poetzl
Quoted 3 lines

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?

the process will be an unique identifier to the namespace, but it might not be easy to use it, so IMHO it might at least make sense to ...

Quoted 7 lines

Especially from userspace. If I want to start a checkpoint on a container, but I have to use the process to identify the container/namespace, well I can't uniquely specify the process by pid anymore...

-serge

Well we first can agree that througout all processes/tasks of a container the namespaces used are the same.
Hence looking at the init_task of each container is sufficient.

Restricting visibility to the default container makes sense to me, because one is not supposed to be able to look into another container.

However, in the global context we do have pid that we can use.

-- Hubertus

#31 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke
Quoted 7 lines · Hubertus Franke

Eric W. Biederman wrote:
So it seems the clone( flags ) is a reasonable approach to create new namespaces. Question is what is the initial state of each namespace? In pidspace we know we should be creating an empty pidmap ! In network, someone suggested creating a loopback device In uts, create "localhost"
Are there examples where we rather inherit ? Filesystem ?

Of course filesystem is already implemented, and does inheret a full copy.

-serge

#32 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 9 lines
Quoted 7 lines

Eric W. Biederman wrote:
So it seems the clone( flags ) is a reasonable approach to create new namespaces. Question is what is the initial state of each namespace? In pidspace we know we should be creating an empty pidmap ! In network, someone suggested creating a loopback device In uts, create "localhost"
Are there examples where we rather inherit ? Filesystem ?

Of course filesystem is already implemented, and does inheret a full copy.

why do we want to use clone()? Just because of its name and flags?
I think it is really strange to fork() to create network context. What has process creation has to do with it?

After all these clone()'s are called, some management actions from host system are still required, to add these IPs/routings/etc. So? Why mess it up? Why not create a separate clean interface for container management?

Kirill

#61 lore raw
In reply to Hubertus Franke
Quoted 5 lines

The pid-namespace (pspace) provides an approach of fully separate the allocation and maintenance of the pids and treating the <pspace,pid> tuple as an entity to uniquely identify a task and vice versa. As a result the logic of lookup can be pushed down the find_task_by_pid() lookup. There are specific cases where the init_task of a container or

Quoted 8 lines

pspace needs to be checked to ensure that signals/waits and alike are properly handled across pspace boundaries. I think this is an intuitive and clean way. It also completely avoids the problem of having to think about all the locations at the user/kernel boundary where a vpid/pid conversion needs to take place. It also avoids the problems that logically vpids and pids are different types and therefore it would have been good to have type checking automatically identify problem spots.
On the negative side, it does require to maintain a pidmap per pidspace.

Additional negative sides:
- full isolation can be inconvinient from containers management point of view. You will need to introduce new modified tools such as top/ps/kill and many many others. You won't be able to strace/gdb processes from the host also.
- overhead when virtualization is off, result is not the same.
- additional args everywhere (stack usage, etc.)

Quoted 5 lines

The vpid approach has the drawbacks of having to identify the conversion spots of all vpid vs. pid semantics. On the otherhand it does take advantage of the fact that no virtualization has to take place until a "container" has been migrated, thus rendering most of the vpid<->pid calls to be noops.

It has some other additional advantages:
- flexible: you can select full isolation or weak is required. I really believe this is very important.

Quoted 2 lines

The container is just an umbrella object that ties every "virtualized" subsystem together.

Yep. And containers were what I wanted to start with actually. Not VPIDs.

Kirill

#56 lore raw
In reply to Hubertus Franke
Quoted 3 lines

My point was to mainly identify the performance culprits and provide an direct access to those "namespaces" for performance reasons. So we all agreed on that we need to do that..

After having looked at Eric's patch, I can tell that he does all these dereferences in quite the same amount.

For example, lot's of skb->sk->host->...
while in OpenVZ it would be econtainer()->... which is essentially current->container->...

So until someone did measurements it looks doubtfull that one solution is better than the another.

Quoted 10 lines

Question now (see other's note as well), should we provide a pointer to each and every namespace in struct task. Seem rather wasteful to me as certain path/namespaces are not exercise heavily.

Having one object "struct container" that still embodies all namespace still seems a reasonable idea.
Otherwise identifying the respective namespace of subsystems will have to go through container->init->subsys_namespace or similar. Not necessarily bad either..

why not simply container->subsys_namespace?

Kirill

#33 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Kirill Korotaev
Quoted 12 lines · Kirill Korotaev
Quoted 10 lines
Quoted 7 lines

Eric W. Biederman wrote:
So it seems the clone( flags ) is a reasonable approach to create new namespaces. Question is what is the initial state of each namespace? In pidspace we know we should be creating an empty pidmap ! In network, someone suggested creating a loopback device In uts, create "localhost"
Are there examples where we rather inherit ? Filesystem ?

Of course filesystem is already implemented, and does inheret a full copy.

Quoted 8 lines

why do we want to use clone()? Just because of its name and flags?
I think it is really strange to fork() to create network context. What has process creation has to do with it?

After all these clone()'s are called, some management actions from host system are still required, to add these IPs/routings/etc. So? Why mess it up? Why not create a separate clean interface for container management?

Kirill

We need a "init" per container, which represents the context of the system represented by the container.
If that is the case, then why not create the container such that
we specify what namespaces need to be new for a container at
the container creation time and initialize them to a well understood state that makes sense (e.g. copy namespace (FS, uts) , new fresh state (pid) ).

Then use the standard syscall to modify state (now "virtualized" through the task->xxx_namespace access ).

Do you see a need to change the namespace of a container after it has been created. I am not referring to the state of the namespace but truely moving to a completely different namespace after the container has been created.

Obviously you seem to have some other usage in mind, beyond what my limited vision can see. Can you share some of those examples, because that would help this discussion along ...

Thanks a 10^6.

-- Hubertus

#57 lore raw
In reply to Kirill Korotaev
Quoted 9 lines · Kirill Korotaev
Quoted 3 lines

My point was to mainly identify the performance culprits and provide an direct access to those "namespaces" for performance reasons. So we all agreed on that we need to do that..

After having looked at Eric's patch, I can tell that he does all these dereferences in quite the same amount.

For example, lot's of skb->sk->host->...
while in OpenVZ it would be econtainer()->... which is essentially current->container->...

Except at that point in time I cannot use current, because it does not have necessarily have the appropriate context. So doubt you could correctly use econtainer there.

Quoted 2 lines

So until someone did measurements it looks doubtfull that one solution is better than the another.

I agree, exactly where we look is a minor matter, unless we can find an instance where there are good reasons for preferring one representation over another. Performance currently does not look a sufficient reason.

My basis for preferring a flat layout is essentially because that is how other similar interfaces are currently implemented in the kernel.

In linux we have a plan9 inspired system call called clone. One of the ideas with clone is that when you create a new task you either share or you don't share resources. Namespaces are one of those resources.

Since we are dealing with concepts that appear to fit the existing model beautifully my feeling is to use the existing model of how things are currently implemented in the kernel.

The only other reason I can think of is namespace implementation independence. If we have an additional structure that we collect up the pointers it feels like it causes unnecessary tying.

The best justification I can think of for putting it all in one structure seems like the current->container current->econtainer thing that comes out of OpenVZ. Currently I have followed the threads enough to know it exists but I yet understand it.

If there is a good reason for the container/econtainer thing I can certainly seem some benefit.

Eric

#35 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Kirill Korotaev
Quoted 7 lines · Kirill Korotaev
Quoted 7 lines
Quoted 7 lines

Eric W. Biederman wrote:
So it seems the clone( flags ) is a reasonable approach to create new namespaces. Question is what is the initial state of each namespace? In pidspace we know we should be creating an empty pidmap ! In network, someone suggested creating a loopback device In uts, create "localhost"
Are there examples where we rather inherit ? Filesystem ?

Quoted 6 lines
Quoted 2 lines

Of course filesystem is already implemented, and does inheret a full copy.

why do we want to use clone()? Just because of its name and flags? I think it is really strange to fork() to create network context. What has process creation has to do with it?

Agreed. Although clones brother unshare takes process creation out of the picture, but otherwise preserves the same interface.

Quoted 4 lines

After all these clone()'s are called, some management actions from host system are still required, to add these IPs/routings/etc.
So? Why mess it up? Why not create a separate clean interface for container management?

If we need additional arguments besides create the thing. We have a clear argument that clone is completely the wrong interface.

However. So far I have not seen an instance where using the existing standard configuration mechanisms from inside the namespace is not the proper way to set things up. The only thing I know that needs to happen from outside is to pass the container a network interface. And if it is a physical interface that is all that must happen.

Eric

#62 lore raw
In reply to Kirill Korotaev
Quoted 4 lines · Kirill Korotaev

Additional negative sides:
- full isolation can be inconvinient from containers management point of view. You will need to introduce new modified tools such as top/ps/kill and many many others. You won't be able to strace/gdb processes from the host also.

Ok. I have to pick a nit here. Needing all new tools top/ps/kill was an artifact of your implementation. Mine does not suffer from it.

Quoted 2 lines

- overhead when virtualization is off, result is not the same.
- additional args everywhere (stack usage, etc.)

Agreed. When using a PID namespace the code always behaves the same. Which is with more arguments than the code used to have. However the code always behaving the same is a tremendous advantage for maintainability.

Quoted 8 lines
Quoted 5 lines

The vpid approach has the drawbacks of having to identify the conversion spots of all vpid vs. pid semantics. On the otherhand it does take advantage of the fact that no virtualization has to take place until a "container" has been migrated, thus rendering most of the vpid<->pid calls to be noops.

It has some other additional advantages:
- flexible: you can select full isolation or weak is required. I really believe this is very important.

I am willing to be convinced but so far there seem to be other solutions like running gdb_stubs inside your container. To address most of the issues.

In a fairly real sense mucking with a container from the outside appears to be a sysadmin layering violation.

Quoted 4 lines
Quoted 3 lines

The container is just an umbrella object that ties every "virtualized" subsystem
together.

Yep. And containers were what I wanted to start with actually. Not VPIDs.

The historical linux approach is to build things out of tasks sharing resources that give the impression of containers not out containers and their children. I am still trying to understand why that model does not work in this instance.

Eric

#36 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke
Quoted 6 lines · Hubertus Franke

So it seems the clone( flags ) is a reasonable approach to create new namespaces. Question is what is the initial state of each namespace? In pidspace we know we should be creating an empty pidmap ! In network, someone suggested creating a loopback device In uts, create "localhost"

"localhost" is wrong. Either copy or set it to the value from system initialization time.

Quoted 2 lines

Are there examples where we rather inherit ? Filesystem ?
Can we iterate the assumption for each subsystem what people thing is right?

I think this needs to happen on a pure subsystem basis as we merge the patches.

Quoted 6 lines

IMHO, there is only a need to refer to a namespace from the global context. Since one will be moving into a new container, but getting out of one could be prohibitive (e.g. after migration)
It does not make sense therefore to know the name of a namespace in a different container.

Quoted 7 lines

The example you used below by using the pid comes natural, because that already limits visibility.

I am still struggling with why we need new sys_calls.
sys_calls already exist for changing certain system parameters (e.g. utsname ) so to me it boils down to identifying a proper initial state when the namespace is created.

Agreed. But we can't always count on everything having a useful state that we can modify from the inside. So it is important to leave the option open at least for those case.

And again there is always the idea that by adding flags we will transform fork or fork+exec into a spaghetti system call. I think that is a reasonable concern.

Also there is always the danger that we run out of clone flags.

Quoted 8 lines
Quoted 2 lines

What I have done which seems easier than creating new names is to refer to the process which has the namespace I want to manipulate.

Is then the idea to only allow the container->init to manipulate or is there need to allow other priviliged processes to perform namespace manipulation?
Also after thinking about it.. why is there a need to have an external name for a namespace ?

Largely it connects to the super chroot usage where you have one sysadmin he has multiple daemons each running in their own environment for isolation purposes. Nothing is installed in the chroot so an attacker that gets in cannot do anything.

Quoted 12 lines
Quoted 11 lines
Quoted 7 lines
Quoted 5 lines

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

Possible to split the lock up now.. one for each pidspace ?

At the moment it is worth thinking about. If the problem isn't so bad that people aren't actively working on it we don't have to solve the problem for a little while, just be aware of it.

Quoted 5 lines

Agree, just need to be sure we can split it up. But you already keep a task list per pid-namespace, so there should be no problem IMHO. If so let's do it now and take it of the table it its as simple as

task_list_lock ::= pspace->task_list_lock

Actually I don't although that could be trivial. But it is the wrong split. The problem is that it is a lock with global effect.

Eric

#37 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke
Quoted 5 lines · Hubertus Franke

IMHO, there is only a need to refer to a namespace from the global context. Since one will be moving into a new container, but getting out of one could be prohibitive (e.g. after migration)
It does not make sense therefore to know the name of a namespace in a different container.

Not sure I agree. What if we are using a private namespace for a vserver, and then we want to create a private namespace in there for a mobile application. Since we're talking about nested namespaces, this should be possible.

Now I believe Eric's code so far would make it so that you can only refer to a namespace from it's *creating* context. Still restrictive, but seems acceptable.

(right?)

-serge

#38 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 9 lines · Serge E. Hallyn
Quoted 5 lines · Hubertus Franke

IMHO, there is only a need to refer to a namespace from the global context. Since one will be moving into a new container, but getting out of one could be prohibitive (e.g. after migration)
It does not make sense therefore to know the name of a namespace in a different container.

Quoted 9 lines

Not sure I agree. What if we are using a private namespace for a vserver, and then we want to create a private namespace in there for a mobile application. Since we're talking about nested namespaces, this should be possible.

Now I believe Eric's code so far would make it so that you can only refer to a namespace from it's *creating* context. Still restrictive, but seems acceptable.

That's what I meant .. as usually used the wrong word.. s/global context/spawing context/g .. because that's the only place where you have a pid to refer to the newly created container !

Quoted 2 lines

(right?)

Yes, seriii .. ahmm serue

Quoted 2 lines

-serge

#34 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke
Quoted 11 lines · Hubertus Franke
Quoted 10 lines · Kirill Korotaev
Quoted 10 lines
Quoted 7 lines

Eric W. Biederman wrote:
So it seems the clone( flags ) is a reasonable approach to create new namespaces. Question is what is the initial state of each namespace? In pidspace we know we should be creating an empty pidmap ! In network, someone suggested creating a loopback device In uts, create "localhost"
Are there examples where we rather inherit ? Filesystem ?

Of course filesystem is already implemented, and does inheret a full copy.

I try to comment on both mails here because I thing that clone() is basically a good interface, but will require some redesign and/or extension ...

Quoted 1 line
Quoted 1 line

why do we want to use clone()?

because it is a natural and existing interface for this purpose at least in Linux-VServer it would work (to some extend). why?

because we aleady use tools like chbind and chcontext which do similar things as chroot, and chroot could, in theory, use clone() and rbind to do it's job ...

Quoted 1 line
Quoted 1 line

Just because of its name and flags?

extending the flags seems natural to me, but the problem might actually be that there are not enough of them left

Quoted 1 line
Quoted 1 line

I think it is really strange to fork() to create network context.

if you look at it as network namespace and sharing existing spaces and/or creating new ones, then clone() and unshare() make pretty much sense there ...

Quoted 1 line
Quoted 1 line

What has process creation has to do with it?

it is a natural interface where you can decide whether to share a space or acquire a new one ... IMHO it would make sense to get trivial userspace tools to create those new spaces in one go, so that the user can use those 'building blocks' to create new spaces whenever she needs

Quoted 2 lines
Quoted 2 lines

After all these clone()'s are called, some management actions from host system are still required, to add these IPs/routings/etc.

not necessarily, for example Linux-VServer uses some kind of 'priviledged' mode, in which the initial guest process can modify various things (like in this case the networking) and setup whatever is required, then, shortly after, giving up those priviledges ...

Quoted 2 lines
Quoted 2 lines

So? Why mess it up?
Why not create a separate clean interface for container management?

I'm not against a clean interface at all, but how would such a 'clean' interface look like?

- a complicated sysfs interface where you write strange values into even stranger places?
- 40 different syscalls to do stuff like adding or removing various parts from the spaces?
- a new ioctl for processes?

Quoted 4 lines
Quoted 1 line

Kirill

We need a "init" per container, which represents the context of the system represented by the container.

that's only partially true, for example Linux-VServer also allows for light-weight guests/containers which do not have a separate init process, just a 'fake' one, so we can save the resources consumed by a separate init process ...

it turns out that this works perfectly fine, even without that fake init, if you teach a few tools like pstree that they should not blindly assume that there is a pid=1 :)

Quoted 5 lines

If that is the case, then why not create the container such that we specify what namespaces need to be new for a container at the container creation time and initialize them to a well understood state that makes sense (e.g. copy namespace (FS, uts) , new fresh state (pid) ).

agreed, but now comes the interesting part, how does such a well understood state look like for all contexts?

obviously the name space makes a complete copy, leading to various issues when you 'try' to get rid of the 'copied' data, like host filesystem mount points and such ...

removing the mounts 'above' a certain chroot() path might seem like a good solution here, but actually it will cause issues when you want to maintain/access a guest/container from the host/parent

leaving all mounts in place will require to either inherit changes from the host/parent to all guests/containers, just to avoid having e.g. /mnt/cdrom mounted in process A, which does not even see it (as it's accessible space starts somewhere else) and therefore being unable to eject the thing, although it is obviously unused

Quoted 2 lines

Then use the standard syscall to modify state (now "virtualized" through the task->xxx_namespace access ).

works as long as you have a handle for that, and actually you do not need one init per guest/container, you need one 'uniquely identified' task 'outside' the container

which in the typical case already makes two of them, the 'handle' task outside and the 'init' task inside ...

Quoted 4 lines

Do you see a need to change the namespace of a container after it has been created. I am not referring to the state of the namespace but truely moving to a completely different namespace after the container has been created.

container itself, probably not, task, definitely yes ... i.e. you definitely want to move between all those spaces given that you are sufficiently priviledged, which is a completely different can of worms ...

Quoted 3 lines

Obviously you seem to have some other usage in mind, beyond what my limited vision can see. Can you share some of those examples, because that would help this discussion along ...

I guess one case where the separate container setup is desired is when you want to keep a container alive even after the processes have seized to exist. for example to visit a 'stopped' guest/context just to do some emergency operations or install some files/packages/etc

IMHO this can easily be solved by keeping the 'handle' process (or whatever handle may be used for the complete context consisting of all spaces) around, even if no process is using those spaces ...

the context as 'collection' of certain namespaces is definitely something which will be required to allow to 'move' into a specific container from the host/parent side, as the parent process obviously does not hold that information.

best,
Herbert

Quoted 6 lines

Thanks a 10^6.

-- Hubertus

#28 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Eric W. Biederman
Quoted 13 lines · On Tue, 07 Feb 2006 20:52:15 -0700 [email protected] (Eric W. Biederman)…
Quoted 6 lines · Serge E. Hallyn

What I tried to do in a proof of concept long ago was to have CLONE_NETNS mean that you get access to all the network devices, but then you could drop/add them. Conceptually I prefer that to getting an empty namespace, but I'm not sure whether there's any practical use where you'd want that...

My observation was that the network stack does not come out cleanly as a namespace unless you adopt the rule that a network device belongs to exactly one network namespace.

Quoted 7 lines

With that rule dealing with the network stack is just a matter of making some currently global variables/data structures per container.

A pain to do the first round but easy to maintain once you are there and the logic of the code doesn't need to change.

Eric

Since a major change risks breaking lots of stuff you would need to have a complete test suite that could be run to show you didn't break anything.

Signature · 2 lines
Stephen Hemminger <[email protected]>
OSDL http://developer.osdl.org/~shemminger
#39 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Serge E. Hallyn
Quoted 3 lines · Serge E. Hallyn

Now I believe Eric's code so far would make it so that you can only refer to a namespace from it's *creating* context. Still restrictive, but seems acceptable.

The same goes for filesystem namespaces. You can't see into random namespaces, just the ones underneath your own. Sounds really reasonable to me.

-- Dave

#63 lore raw
In reply to Kirill Korotaev
Quoted 4 lines · Kirill Korotaev

- full isolation can be inconvinient from containers management point of view. You will need to introduce new modified tools such as top/ps/kill and many many others. You won't be able to strace/gdb processes from the host also.

I'd like to put a theory out there: the more isolation we perform, the easier checkpointing and migration become to guarantee.

Agree? Disagree?

But, full isolation is hard to code. The right approach is very likely somewhere in the middle where we require some things to happen underneath us. For instance, requiring that the filesystem be made consistent if a container is moved across systems.

-- Dave

#64 lore raw
Quoted 10 lines · Dave Hansen
Quoted 4 lines · Kirill Korotaev

- full isolation can be inconvinient from containers management point of view. You will need to introduce new modified tools such as top/ps/kill and many many others. You won't be able to strace/gdb processes from the host also.

I'd like to put a theory out there: the more isolation we perform, the easier checkpointing and migration become to guarantee.

Agree? Disagree?

Agree. But that does not address the reasons OpenVZ and Vserver exist.

Quoted 1 line

But, full isolation is hard to code.

Disagree. If you limit your self to just changing the code that translates from names to objects it is a very narrow slice of code, and there are very few surprises. There is a lot of grunt work involved but it easy to tell if you got everything and did it correctly.

Other approaches are more adhoc, take short cuts, and seem prone to missing the corner cases.

Quoted 4 lines

The right approach is very likely
somewhere in the middle where we require some things to happen underneath us. For instance, requiring that the filesystem be made consistent if a container is moved across systems.

Possibly. That is very out from where we are at the moment. Let's get the isolation and see where we are at.

Eric

#40 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Dave Hansen
Quoted 8 lines · Dave Hansen
Quoted 3 lines · Serge E. Hallyn

Now I believe Eric's code so far would make it so that you can only refer to a namespace from it's *creating* context. Still restrictive, but seems acceptable.

The same goes for filesystem namespaces. You can't see into random namespaces, just the ones underneath your own. Sounds really reasonable to me.

Hmmm? I suspect I'm misreading what you're saying, but to be clear:

Let's say I start a screen session. In one of those shells, I clone, specify CLONE_NEWNS, and exec a shell. now i do a bunch of mounting. Other shells in the screen session won't see the results of those mounts, and if i ctrl-d, the shell which started the screen session can't either. Each of these is in the "parent filesystem namespace".

OTOH, shared subtrees specified in the parent shell could make it such that the parent ns, but not others, see the results. Is that what you're referring to?

thanks,
-serge

#41 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke
Quoted 12 lines · Hubertus Franke
Quoted 11 lines · Eric W. Biederman
Quoted 9 lines · Hubertus Franke
Quoted 4 lines · Eric W. Biederman

3) How do we refer to namespaces and containers when we are not members?
- Do we refer to them indirectly by processes or other objects that we can see and are members?
- Do we assign some kind of unique id to the containers?

Quoted 8 lines
Quoted 2 lines

What I have done which seems easier than creating new names is to refer to the process which has the namespace I want to manipulate.

Is then the idea to only allow the container->init to manipulate or is there need to allow other priviliged processes to perform namespace manipulation?
Also after thinking about it.. why is there a need to have an external name for a namespace ?

There are several cases.

Passing network devices to a childs namespace, as usually the loopback interface is not enough.

Monitoring the namespace from outside, so among other things you aren't required to checkpoint and migrate your monitoring daemon.

There are several other control and monitoring operations that I am not quite as familiar. One of them is the vserver idea of entering a guest.

To expand on things a little bit. If we have interfaces that take strings we can refer to an arbitrary child process as pid/pid/pid/.... So we should not be limited to what is at the init of the container. If that proves desirable.

Permissions checks for most of these operations require some serious thinking before they are merged.

Eric

#69 lore raw
In reply to Kirill Korotaev
Quoted 19 lines · Kirill Korotaev

Hello,

I tried to take into account all the comments from you guys (thanks a lot for them!) and prepared a new version of virtualization patches. I will send only 4 patches today, just not to overflow everyone and keep it clear/tidy/possible to review.

This patch introduces some abstract container kernel structure and a number of operations on it.

The important properties of the proposed container implementation:
- each container has unique ID in the system
- each process in the kernel can belong to one container only
- effective container pointer (econtainer()) is used on the task to avoid insertion of additional argument "container" to all functions where it is required.
- kernel compilation with disabled virtualization should result in old good linux kernel

Quoted 4 lines

Patches following this one will be used for virtualization of the kernel resources based on this container infrastructure, including those VPID patches I sent before. Every virtualized resource can be given separate config option if needed (just give me to know if it is desired).

After having digested this I think there is something sane with regard to this container idea. I still think the implementation is totally wrong but there is a potential problem the basic idea solves.

In the traditional linux/plan9 style of namespaces the question is which resources different tasks share, and we pass clone bits to determine what we want to share and what we don't want to share. Semantically this is very clean and allows for a great deal of flexibility. However as the flexibility increases we get more code in do_fork more reference counts and ultimately the performance decreases. In addition it is not common for us to change which resources we share.

So our traditional route is flexible but it does not optimize the common case where we share all of the same things. Containers can potentially optimize that case. So long as anyone can create a new container even someone inside a container we do not loose flexibility.

To deal with networking there are currently a significant number of variables with static storage duration. Making those variables global and placing them in structures is neither as efficient as it could be nor is it as maintainable as it should be. Other subsystems have similar problems.

So if we put econtainer in struct thread_info and optimize it like we do current, create an interface to variables similar to DEFINE_PER_CPU, create a syscall interface similar to clone, and show that by doing this we don't loose flexibility, then it looks like a good idea.

If we can't do better than the current clone model of shared resources then this model feels like gratuitous change.

Eric

#70 lore raw
Quoted 5 lines · Eric W. Biederman

To deal with networking there are currently a significant number of variables with static storage duration. Making those variables global and placing them in structures is neither as efficient as it could be nor is it as maintainable as it should be. Other subsystems have similar problems.

BTW, there is another solution, which you may or may not consider to be clean.

That is to load a separate copy of the subsystem (code and data) as a module when you want a new instance of it. The code doesn't change, but you probably have to move it around some and provide some sort of interface to it.

I did this to the scheduler last year - see

	http://marc.theaimsgroup.com/?l=linux-kernel&m=111404726721747&w=2

				Jeff
#71 lore raw
Quoted 7 lines · Jeff Dike
Quoted 5 lines · Eric W. Biederman

To deal with networking there are currently a significant number of variables with static storage duration. Making those variables global and placing them in structures is neither as efficient as it could be nor is it as maintainable as it should be. Other subsystems have similar problems.

Quoted 7 lines

BTW, there is another solution, which you may or may not consider to be clean.

That is to load a separate copy of the subsystem (code and data) as a module when you want a new instance of it. The code doesn't change, but you probably have to move it around some and provide some sort of interface to it.

There are some real drawbacks to that. Basically as things are structured you only want multiple copies of the user facing data structures. Pretty much everything else really does remain the same.

Quoted 2 lines

I did this to the scheduler last year - see

	http://marc.theaimsgroup.com/?l=linux-kernel&m=111404726721747&w=2

I will take a look but I don't expect to find anything.

Eric

#49 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Eric W. Biederman
Quoted 7 lines · Eric W. Biederman

I think I can boil the discussion down into some of the fundamental questions that we are facing.

Currently everyone seems to agree that we need something like my namespace concept that isolates multiple resources.

We need these for
UIDS
FILESYSTEM

I have one suggestion for this (it also covers capabilities to a certain extent). Could we use the kernel credentials system to abstract away the concept of a single UID/GID? We currently have uid, euid, gid, egid, groups, fsid. I'm thinking that there would be virtualized UID tables to determine ownership of processes/SHM/etc.

Each process would have a (uid_container,uid) pair (or similar) as its "uid" and likewise for gid. Then the ability to send signals to any given (uid_container,uid) or (gid_container,gid) pair would be given by keys in the kernel keyring indexed by the "uid_container" part and containing the "uid" part (or maybe just a pointer).

Likewise the filesystem access could be virtualized by using uid and gid keys in the kernel keyring indexed by vfsmount (Not superblock, so that it would be possible to have different UID representations on different mounts/parts of the same filesystem).

I'm guessing that the performance implications of the above would not be quite so nice, as it would put a lot of code in the fastpath, but I would guess that it might be possible to use the existing fields for processes without any virtualization needs.

Cheers,
Kyle Moffett

--
There is no way to make Linux robust with unreliable memory subsystems, sorry. It would be like trying to make a human more robust with an unreliable O2 supply. Memory just has to work.
-- Andi Kleen

#50 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 28 lines · Kyle Moffett
Quoted 9 lines · Eric W. Biederman

I think I can boil the discussion down into some of the fundamental questions that we are facing.

Currently everyone seems to agree that we need something like my namespace concept that isolates multiple resources.

We need these for
UIDS
FILESYSTEM

I have one suggestion for this (it also covers capabilities to a certain extent). Could we use the kernel credentials system to abstract away the concept of a single UID/GID? We currently have uid, euid, gid, egid, groups, fsid. I'm thinking that there would be virtualized UID tables to determine ownership of processes/SHM/etc.

Each process would have a (uid_container,uid) pair (or similar) as its "uid" and likewise for gid. Then the ability to send signals to any given (uid_container,uid) or (gid_container,gid) pair would be given by keys in the kernel keyring indexed by the "uid_container" part and containing the "uid" part (or maybe just a pointer).

Likewise the filesystem access could be virtualized by using uid and gid keys in the kernel keyring indexed by vfsmount (Not superblock, so that it would be possible to have different UID representations on different mounts/parts of the same filesystem).

Quoted 4 lines

I'm guessing that the performance implications of the above would not be quite so nice, as it would put a lot of code in the fastpath, but I would guess that it might be possible to use the existing fields for processes without any virtualization needs.

At least for signal sending it looks like it would be easier to just compare the pointers to struct user. At least in that context it looks like it would be as cheap as what we are doing now. I just don't know where to find a struct user for the euid, or is it the normal uid.

Eric

#72 lore raw
In reply to Jeff Dike
Quoted 10 lines
Quoted 5 lines

To deal with networking there are currently a significant number of variables with static storage duration. Making those variables global and placing them in structures is neither as efficient as it could be nor is it as maintainable as it should be. Other subsystems have similar problems.

BTW, there is another solution, which you may or may not consider to be clean.

Quoted 7 lines

That is to load a separate copy of the subsystem (code and data) as a module when you want a new instance of it. The code doesn't change, but you probably have to move it around some and provide some sort of interface to it.

I did this to the scheduler last year - see

	http://marc.theaimsgroup.com/?l=linux-kernel&m=111404726721747&w=2

It's really interesting!
Have you tested fairness of your solution and it's performance overhead?

Kirill

#73 lore raw
Quoted 4 lines · Kirill Korotaev
Quoted 2 lines

I did this to the scheduler last year - see

	http://marc.theaimsgroup.com/?l=linux-kernel&m=111404726721747&w=2

It's really interesting!
Have you tested fairness of your solution and it's performance overhead?

What do you mean by fairness, exactly?

As for its overhead, I just got it working inside UML. I tried it on x86_64, but something was wrong with the low-level switching stuff, and the machine hung whenever a guest scheduler process tried to run. So, I never got any real measurements.

I had better things to do, so I dropped this and went back to them.

Jeff

#74 lore raw
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I did this to the scheduler last year - see

	http://marc.theaimsgroup.com/?l=linux-kernel&m=111404726721747&w=2

It's really interesting!
Have you tested fairness of your solution and it's performance overhead?

What do you mean by fairness, exactly?

I mean how CPU time is distributed not only in the case of CPU hogs. For example, when 2 tasks do cyclic 1 byte transfer via pipe. one of them is awake, while another goes to sleep. If both are in one container, will they behave like a CPU hog?

Quoted 4 lines

As for its overhead, I just got it working inside UML. I tried it on x86_64, but something was wrong with the low-level switching stuff, and the machine hung whenever a guest scheduler process tried to run. So, I never got any real measurements.

It's a pity... :( We have fair CPU scheduler in OpenVZ project, so it's quite an interesting approach for us.

Kirill

#76 lore raw
In reply to Jeff Dike
Quoted 17 lines · Jeff Dike
Quoted 5 lines · Eric W. Biederman

To deal with networking there are currently a significant number of variables with static storage duration. Making those variables global and placing them in structures is neither as efficient as it could be nor is it as maintainable as it should be. Other subsystems have similar problems.

BTW, there is another solution, which you may or may not consider to be clean.

That is to load a separate copy of the subsystem (code and data) as a module when you want a new instance of it. The code doesn't change, but you probably have to move it around some and provide some sort of interface to it.

Quoted 5 lines

I did this to the scheduler last year - see

	http://marc.theaimsgroup.com/?l=linux-kernel&m=111404726721747&w=2

				Jeff

Jeff, interesting, but won't that post some serious scalability issue? Imaging 100s of container/namespace ?

The namespace is mainly there to identify which data needs to be private when multiple instances of a subsystem are considered and encapsulate that data in an object/datastructure !

-- Hubertus

#77 lore raw
Quoted 2 lines · Hubertus Franke

Jeff, interesting, but won't that post some serious scalability issue? Imaging 100s of container/namespace ?

In terms of memory?

Running size on sched.o gives me this on x86_64:
text data bss dec hex filename
35685 6880 28800 71365 116c5 sched.o

and on i386 (actually UML/i386)

text data bss dec hex filename
10010 36 2504 12550 3106 obj/kernel/sched.o

I'm not sure why there's such a big difference, but 100 instances adds a meg or two (or three) to the kernel. This overstates things a bit because there are things in sched.c which wouldn't be duplicated, like the system calls.

How big a deal is that on a system which you plan to have 100s of containers on anyway?

It's heavier than your namespaces, but does have the advantage that it imposes no cost when it's not being used.

Quoted 3 lines

The namespace is mainly there to identify which data needs to be private when multiple instances of a subsystem are considered and encapsulate that data in an object/datastructure !

Sure, and that's a fine approach. It's just not the only one.

Jeff

#75 lore raw
In reply to Kirill Korotaev
Quoted 4 lines · Kirill Korotaev

I mean how CPU time is distributed not only in the case of CPU hogs. For example, when 2 tasks do cyclic 1 byte transfer via pipe. one of them is awake, while another goes to sleep.
If both are in one container, will they behave like a CPU hog?

So if you have a container with processes taking turns running, so that there is always a running process, does the container as a whole act as a CPU hog?

I'll see if I can resurrect it and see.

Jeff

#79 lore raw
In reply to Eric W. Biederman

Ok digesting this some more I don't see a glaring case in the code where it is set_econtainer(...); do stuff ; restore_econtainer(..); has a tremendous advantage at the moment. Plus it removes the opportunity to catch old code that deals with multiple contexts at compile time, and has to be done by pain staking code review.

That aside the truly important thing I realized is that for the optimization effects it is an implementation detail. Batched reference counting of shared resources is something we can go back in and add at any time. It will take a little refactoring to do but it something that can be done without changing the user space API.

So since individual namespace pointers in the task_struct are simpler and have the required flexibility. I think we can table this part of the discussion for now.

The other detail I brought up which was most interesting was the marking of specific variables that would go into a container or namespace. Well as long as it is implemented on the namespace level that is still possible, and may even be beneficial.

Eric

#78 lore raw
In reply to Jeff Dike
Quoted 12 lines · Jeff Dike
Quoted 2 lines · Hubertus Franke

Jeff, interesting, but won't that post some serious scalability issue? Imaging 100s of container/namespace ?

In terms of memory? Running size on sched.o gives me this on x86_64:
text data bss dec hex filename
35685 6880 28800 71365 116c5 sched.o

and on i386 (actually UML/i386)

text data bss dec hex filename
10010 36 2504 12550 3106 obj/kernel/sched.o

Quoted 7 lines

I'm not sure why there's such a big difference, but 100 instances adds a meg or two (or three) to the kernel. This overstates things a bit because there are things in sched.c which wouldn't be duplicated, like the system calls.

How big a deal is that on a system which you plan to have 100s of containers on anyway?

Quite a big deal. You might have 2Gigs of main memory, but your CPU is unlikely to be more than a Megabyte in close reach. A meg or two of scheduler data and code means that your L1 and L2 cache will be cycling every scheduler round; which is OK if you have very short runqueues but as you get more and more processes it will really start to hurt.

Remember, systems today are memory bound and anything you can do to reduce the amount of time the system sits around waiting for memory to fetch, the better.

Compare that to the Token Bucket Scheduler of Linux-VServer; a tiny struct for each process umbrella, that will generally fit in one or two cachelines, to which the scheduling support adds four ints and a spinlock. With this it achieves fair CPU scheduling between vservers.

Sam.

#51 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Eric W. Biederman
Quoted 15 lines · [email protected] (Eric W. Biederman) writes:

I think I can boil the discussion down into some of the fundamental questions that we are facing.

Currently everyone seems to agree that we need something like my namespace concept that isolates multiple resources.

We need these for
PIDS
UIDS
SYSVIPC
NETWORK
UTSNAME
FILESYSTEM
etc.

Quoted 5 lines

The questions seem to break down into:
1) Where do we put the references to the different namespaces?
- Do we put the references in a struct container that we reference from
struct task_struct?
- Do we put the references directly in struct task_struct?

Answer in the task_struct. It is the simplest and most flexible route and the other implementations are still possible.

Quoted 6 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags?
(Plan 9 style)
- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?
- Do we in addition add syscalls to manipulate containers generically?

The answer seems to be we decide on a per namespace basis with additional syscalls being mandatory if we have any additional data to pass.

Quoted 1 line

3) How do we refer to namespaces and containers when we are not members?

I have seen no arguments against referring to namespaces or containers by global ids. So it seems we do not need a container id.

Quoted 2 lines

4) How do we implement each of these namespaces? Besides being maintainable are there other constraints?

Largely quite. But I have not heard additional constraints.

Quoted 5 lines

5) How do we control the resource inside a namespace starting from a process that is outside of that namespace?
- The filesystem mount namespace gave an interesting answer. So it is quite possible other namespaces will give equally interesting and surprising answers.

Not yet resolved, but a bit of speculation.

Quoted 5 lines

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

A little discussion. At the level of the last few cache line I think this needs to be addressed when we merge. Simply not messing up existing optimizations sounds like a good initial target. Basically at this stage trying hard would be a premature optimization.

Quoted 4 lines

7) How do we allow a process inside a container to create containers for it's children?
- In general this is trivial but there are a few ugly issues here.

This look mostly like something to be discussed when we merge namespaces. But as long as we keep it in mind it is easy.

Eric

#42 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

In reply to Hubertus Franke
Quoted 11 lines
Quoted 5 lines

The questions seem to break down into:
1) Where do we put the references to the different namespaces?
- Do we put the references in a struct container that we reference
from struct task_struct?
- Do we put the references directly in struct task_struct?

You "cache" task_struct->container->hotsubsys under task_struct->hotsubsys.
We don't change containers other then at clone time, so no coherency issue here !!!! Which subsystems pointers to "cache", should be agreed by the experts, but first approach should always not to cache and go through the container.

agreed. I see no much reason to cache it and make tons of the same pointers in all the tasks. Only if needed. Also, in OpenVZ container has many fields intergrated inside, so there is no additional dereference, but task->container->subsys_field

Quoted 3 lines
Quoted 2 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags? (Plan 9 style)

Like that approach .. flexible .. particular when one has well specified namespaces.

mmm, how do you plan to pass additional flags to clone()? e.g. strong or weak isolation of pids?

another questions:
how do you plan to meet the dependancies between namespaces? e.g. conntracks require netfilters to be initialized. network requires sysctls and proc to be initialized and so on. do you propose to track all this in clone()? huh...

Quoted 2 lines
Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

can be so...

Quoted 8 lines
Quoted 7 lines

- Do we in addition add syscalls to manipulate containers generically?

I don't think having a single system call to create a container and
a new
instance of each namespace is reasonable as that does not give us a path into the future when we create yet another namespace.

Agreed.

why do you think so?
this syscalls will start handling this new namespace and that's all. this is not different from many syscalls approach.

Quoted 10 lines
Quoted 3 lines

4) How do we implement each of these namespaces? Besides being maintainable are there other constraints?

Good question... at least with PID and FS two are there ..

Quoted 6 lines

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

this is nothing compared to hierarchy operations.
BTW, heirarchy also introduces complicated resource accounting, sometimes making it even impossible.

Kirill

#43 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 13 lines · Kirill Korotaev
Quoted 7 lines
Quoted 5 lines

The questions seem to break down into:
1) Where do we put the references to the different namespaces?
- Do we put the references in a struct container that we reference
from struct task_struct?
- Do we put the references directly in struct task_struct?

Quoted 6 lines

You "cache" task_struct->container->hotsubsys under
task_struct->hotsubsys.
We don't change containers other then at clone time, so no coherency issue here !!!!
Which subsystems pointers to "cache", should be agreed by the experts, but first approach should always not to cache and go through the container.

Quoted 5 lines

agreed. I see no much reason to cache it and make tons of the same pointers in all the tasks. Only if needed.

Also, in OpenVZ container has many fields intergrated inside, so there is no additional dereference, but task->container->subsys_field

as does Linux-VServer currently, but do you have any proof that putting all the fields together in one big structure actually has any (dis)advantage over separate structures?

Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

2) What is the syscall interface to create these namespaces?
- Do we add clone flags? (Plan 9 style)

Like that approach .. flexible .. particular when one has well specified namespaces.

mmm, how do you plan to pass additional flags to clone()? e.g. strong or weak isolation of pids?

do you really have to pass them at clone() time? would shortly after be more than enough? what if you want to change those properties later?

Quoted 5 lines

another questions:
how do you plan to meet the dependancies between namespaces? e.g. conntracks require netfilters to be initialized. network requires sysctls and proc to be initialized and so on. do you propose to track all this in clone()? huh...

this is missing isolation/virtualization, and I guess it has to be done to make those spaces useful ...

Quoted 11 lines
Quoted 2 lines
Quoted 2 lines

- Do we add a syscall (similar to setsid) per namespace? (Traditional unix style)?

can be so...

Quoted 7 lines
Quoted 7 lines

- Do we in addition add syscalls to manipulate containers generically?

I don't think having a single system call to create a container and a new instance of each namespace is reasonable as that does not give us a path into the future when we create yet another namespace.

Quoted 5 lines
Quoted 1 line

Agreed.

why do you think so?

this syscalls will start handling this new namespace and that's all. this is not different from many syscalls approach.

well, let's defer the 'how amny syscalls' issue to a later time, when we know what we want to implement :)

Quoted 11 lines
Quoted 10 lines
Quoted 3 lines

4) How do we implement each of these namespaces? Besides being maintainable are there other constraints?

Good question... at least with PID and FS two are there ..

Quoted 6 lines

6) How do we do all of this efficiently without a noticeable impact on performance?
- I have already heard concerns that I might be introducing cache line bounces and thus increasing tasklist_lock hold time. Which on big way systems can be a problem.

Quoted 3 lines

this is nothing compared to hierarchy operations. BTW, heirarchy also introduces complicated resource accounting, sometimes making it even impossible.

well, depends how you do it ...

best,
Herbert

Quoted 1 line

Kirill

#44 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 4 lines

as does Linux-VServer currently, but do you have any proof that putting all the fields together in one big structure actually has any (dis)advantage over separate structures?

have no proof and don't mind if there are many pointers. Though this doesn't look helpful to me as well.

Quoted 5 lines
Quoted 2 lines

mmm, how do you plan to pass additional flags to clone()? e.g. strong or weak isolation of pids?

do you really have to pass them at clone() time? would shortly after be more than enough? what if you want to change those properties later?

I don't think it is always suiatable to do configuration later.
We had races in OpenVZ on VPS create/stop against exec/enter etc. (even introduced flag is_running). So I have some experience to believe it will be painfull place.

Quoted 4 lines
Quoted 2 lines

this syscalls will start handling this new namespace and that's all. this is not different from many syscalls approach.

well, let's defer the 'how amny syscalls' issue to a later time, when we know what we want to implement :)

agreed.

Kirill

#45 lore raw

Re: The issues for agreeing on a virtualization/namespaces implementation.

Quoted 7 lines · Kirill Korotaev
Quoted 4 lines

as does Linux-VServer currently, but do you have any proof that putting all the fields together in one big structure actually has any (dis)advantage over separate structures?

have no proof and don't mind if there are many pointers. Though this doesn't look helpful to me as well.

well, my point is just that we don't know yet so we should not favor one over the other, just because somebody did it like that and it didn't hurt :)

Quoted 11 lines
Quoted 6 lines
Quoted 2 lines

mmm, how do you plan to pass additional flags to clone()? e.g. strong or weak isolation of pids?

do you really have to pass them at clone() time? would shortly after be more than enough? what if you want to change those properties later?

I don't think it is always suiatable to do configuration later. We had races in OpenVZ on VPS create/stop against exec/enter etc. (even introduced flag is_running).
So I have some experience to believe it will be painfull place.

well, Linux-VServer uses a state called 'setup' which allows to change all kinds of things before the guest can be entered, this state is changed as the last operation of the setup, which in turn drops all the capabilities and makes the guest visible to the outside ...

works quite well and seems to be free of those races you mentioned ...

Quoted 5 lines
Quoted 4 lines
Quoted 2 lines

this syscalls will start handling this new namespace and that's all. this is not different from many syscalls approach.

well, let's defer the 'how amny syscalls' issue to a later time, when we know what we want to implement :)

agreed.

btw, maybe it's just me, but would it be possible to do the email quoting like this:

Quoted 1 line
Quoted 1 line
Quoted 1 line

Text

instead of

Quoted 1 line
Quoted 1 line
Quoted 1 line

Text

TIA,
Herbert

Quoted 1 line

Kirill