Skip to contentSkip to conversation

Generic Process Containers

PATCH In review [email protected] 3 participants 12 messages Oct 20, 2006 – Nov 7, 2006

Subsystems (10)
lore
Share & tools
lore
#0 lore raw

cover Generic Process Containers

--

This is an update to my generic containers patch, with the following changes:

- ported to 2.6.19-rc2
- CONFIG_CPUSETS_LEGACY_API option maintains the existing cpusets userspace API
- support for fork/exit callbacks

Patch 6 contains a port of the interesting bits of ResGroups to run over generic containers, along with the example ResGroups numtasks patch. It's not intended to be actually applied with this patch set, but is an example of how other in-kernel systems might use generic containers.

(This time built with multiple compilers and architectures)

-------------------------------------

There have recently been various proposals floating around for resource management/accounting subsystems in the kernel, including Res Groups, User BeanCounters and others. These all need the basic abstraction of being able to group together multiple processes in an aggregate, in order to track/limit the resources permitted to those processes, and all implement this grouping in different ways.

Already existing in the kernel is the cpuset subsystem; this has a process grouping mechanism that is mature, tested, and well documented (particularly with regards to synchronization rules).

This patchset extracts the process grouping code from cpusets into a generic container system, and makes the cpusets code a client of the container system.

It also provides a very simple additional container subsystem to do per-container CPU usage accounting; this is primarily to demonstrate use of the container subsystem API, but is useful in its own right.

The change is implemented in five stages:

1) extract the process grouping code from cpusets into a standalone system

2) remove the process grouping code from cpusets and hook into the container system

3) convert the container system to present a generic API, and make cpusets a client of that API

4) add a simple CPU accounting container subsystem as an example

5) add support for fork/exit callbacks iff some subsystem is interested in them

The intention is that the various resource management efforts can also become container clients, with the result that:

- the userspace APIs are (somewhat) normalised

- it's easier to test out e.g. the ResGroups CPU controller in conjunction with the UBC memory controller

- the additional kernel footprint of any of the competing resource management systems is substantially reduced, since it doesn't need to provide process grouping/containment, hence improving their chances of getting into the kernel

Possible TODOs include:

- define a convention for populating the per-container directories so that different subsystems don't clash with one another

- provide higher-level primitives (e.g. an easy interface to seq_file) for files registered by subsystems, or potentially convert to use configfs

--

#1 lore raw

PATCH 1/6 Generic container system abstracted from cpusets code

This patch creates a generic process container system based on (and parallel top) the cpusets code. At a coarse level it was created by copying kernel/cpuset.c, doing s/cpuset/container/g, and stripping out any code that was cpuset-specific rather than applicable to any process container subsystem.

---
 Documentation/containers.txt |  229 +++++++
 fs/proc/base.c               |    7 
 include/linux/container.h    |   96 +++
 include/linux/sched.h        |    5 
 init/Kconfig                 |    9 
 init/main.c                  |    3 
 kernel/Makefile              |    1 
 kernel/container.c           | 1343 +++++++++++++++++++++++++++++++++++++++++++
 kernel/exit.c                |    2 
 kernel/fork.c                |    3 
 10 files changed, 1697 insertions(+), 1 deletion(-)
fs/proc/base.c +7 -0
Index: container-2.6.19-rc2/fs/proc/base.c
===================================================================
@@ -69,6 +69,7 @@
 #include <linux/ptrace.h>
 #include <linux/seccomp.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/audit.h>
 #include <linux/poll.h>
 #include <linux/nsproxy.h>
@@ -1780,6 +1781,9 @@ static struct pid_entry tgid_base_stuff[
 #ifdef CONFIG_SCHEDSTATS
 	INF("schedstat",  S_IRUGO, pid_schedstat),
 #endif
+#ifdef CONFIG_CONTAINERS
+	REG("container",  S_IRUGO, container),
+#endif
 #ifdef CONFIG_CPUSETS
 	REG("cpuset",     S_IRUGO, cpuset),
 #endif
@@ -2054,6 +2058,9 @@ static struct pid_entry tid_base_stuff[]
 #ifdef CONFIG_SCHEDSTATS
 	INF("schedstat", S_IRUGO, pid_schedstat),
 #endif
+#ifdef CONFIG_CONTAINERS
+	REG("container",  S_IRUGO, container),
+#endif
 #ifdef CONFIG_CPUSETS
 	REG("cpuset",    S_IRUGO, cpuset),
 #endif
include/linux/container.h new file +96 -0

96 lines not on this page · raw message

include/linux/sched.h +4 -1
Index: container-2.6.19-rc2/include/linux/sched.h
===================================================================
@@ -720,8 +720,8 @@ extern unsigned int max_cache_size;
 
 
 struct io_context;			/* See blkdev.h */
+struct container;
 struct cpuset;
-
 #define NGROUPS_SMALL		32
 #define NGROUPS_PER_BLOCK	((int)(PAGE_SIZE / sizeof(gid_t)))
 struct group_info {
@@ -1007,6 +1007,9 @@ struct task_struct {
 	int cpuset_mems_generation;
 	int cpuset_mem_spread_rotor;
 #endif
+#ifdef CONFIG_CONTAINERS
+	struct container *container;
+#endif
 	struct robust_list_head __user *robust_list;
 #ifdef CONFIG_COMPAT
 	struct compat_robust_list_head __user *compat_robust_list;
init/Kconfig +9 -0
Index: container-2.6.19-rc2/init/Kconfig
===================================================================
@@ -237,6 +237,15 @@ config IKCONFIG_PROC
 	  This option enables access to the kernel configuration file
 	  through /proc/config.gz.
 
+config CONTAINERS
+	bool "Container support"
+	help
+	  This option will let you create and manage process containers,
+	  which can be used to aggregate multiple processes, e.g. for
+	  the purposes of resource tracking.
+
+	  Say N if unsure
+
 config CPUSETS
 	bool "Cpuset support"
 	depends on SMP
init/main.c +3 -0
Index: container-2.6.19-rc2/init/main.c
===================================================================
@@ -38,6 +38,7 @@
 #include <linux/writeback.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/efi.h>
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
@@ -567,6 +568,7 @@ asmlinkage void __init start_kernel(void
 	}
 #endif
 	vfs_caches_init_early();
+	container_init_early();
 	cpuset_init_early();
 	mem_init();
 	kmem_cache_init();
@@ -597,6 +599,7 @@ asmlinkage void __init start_kernel(void
 #ifdef CONFIG_PROC_FS
 	proc_root_init();
 #endif
+	container_init();
 	cpuset_init();
 	taskstats_init_early();
 	delayacct_init();
kernel/container.c new file +1343 -0

1,343 lines not on this page · raw message

kernel/exit.c +2 -0
Index: container-2.6.19-rc2/kernel/exit.c
===================================================================
@@ -30,6 +30,7 @@
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/syscalls.h>
 #include <linux/signal.h>
 #include <linux/posix-timers.h>
@@ -922,6 +923,7 @@ fastcall NORET_TYPE void do_exit(long co
 	__exit_fs(tsk);
 	exit_thread();
 	cpuset_exit(tsk);
+	container_exit(tsk);
 	exit_keys(tsk);
 
 	if (group_dead && tsk->signal->leader)
kernel/fork.c +3 -0
Index: container-2.6.19-rc2/kernel/fork.c
===================================================================
@@ -31,6 +31,7 @@
 #include <linux/capability.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/security.h>
 #include <linux/swap.h>
 #include <linux/syscalls.h>
@@ -1053,6 +1054,7 @@ static struct task_struct *copy_process(
 	p->io_context = NULL;
 	p->io_wait = NULL;
 	p->audit_context = NULL;
+	container_fork(p);
 	cpuset_fork(p);
 #ifdef CONFIG_NUMA
  	p->mempolicy = mpol_copy(p->mempolicy);
@@ -1288,6 +1290,7 @@ bad_fork_cleanup_policy:
 bad_fork_cleanup_cpuset:
 #endif
 	cpuset_exit(p);
+	container_exit(p);
 bad_fork_cleanup_delays_binfmt:
 	delayacct_tsk_free(p);
 	if (p->binfmt)
kernel/Makefile +1 -0
Index: container-2.6.19-rc2/kernel/Makefile
===================================================================
@@ -36,6 +36,7 @@ obj-$(CONFIG_PM) += power/
 obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o
 obj-$(CONFIG_KEXEC) += kexec.o
 obj-$(CONFIG_COMPAT) += compat.o
+obj-$(CONFIG_CONTAINERS) += container.o
 obj-$(CONFIG_CPUSETS) += cpuset.o
 obj-$(CONFIG_IKCONFIG) += configs.o
 obj-$(CONFIG_STOP_MACHINE) += stop_machine.o
Documentation/containers.txt new file +229 -0

229 lines not on this page · raw message

--

#2 lore raw

PATCH 2/6 Cpusets hooked into containers

In reply to [email protected]

This patch removes the process grouping code from the cpusets code, instead hooking it into the generic container system. This temporarily adds cpuset-specific code in kernel/container.c, which is removed by the next patch in the series.

---
 Documentation/cpusets.txt |   80 +-
 fs/super.c                |    5 
 include/linux/container.h |    7 
 include/linux/cpuset.h    |   25 
 include/linux/fs.h        |    2 
 include/linux/sched.h     |    4 
 init/Kconfig              |   19 
 kernel/container.c        |  105 +++
 kernel/cpuset.c           | 1321 +++++-----------------------------------------
 kernel/exit.c             |    2 
 kernel/fork.c             |    7 
 mm/oom_kill.c             |    6 
 12 files changed, 317 insertions(+), 1266 deletions(-)
include/linux/container.h +7 -0
Index: container-2.6.19-rc2/include/linux/container.h
===================================================================
@@ -47,6 +47,10 @@ struct container {
 
 	struct container *parent;	/* my parent */
 	struct dentry *dentry;		/* container fs entry */
+
+#ifdef CONFIG_CPUSETS
+	struct cpuset *cpuset;
+#endif
 };
 
 /* struct cftype:
@@ -79,6 +83,9 @@ struct cftype {
 int container_add_file(struct container *cont, const struct cftype *cft);
 
 int container_is_removed(const struct container *cont);
+#ifdef CONFIG_CPUSETS_LEGACY_API
+void container_set_release_agent_path(const char *path);
+#endif
 
 #else /* !CONFIG_CONTAINERS */
 
include/linux/cpuset.h +13 -12
Index: container-2.6.19-rc2/include/linux/cpuset.h
===================================================================
@@ -11,16 +11,15 @@
 #include <linux/sched.h>
 #include <linux/cpumask.h>
 #include <linux/nodemask.h>
+#include <linux/container.h>
 
 #ifdef CONFIG_CPUSETS
 
-extern int number_of_cpusets;	/* How many cpusets are defined in system? */
+extern int number_of_cpusets;  /* How many cpusets are defined in system? */
 
 extern int cpuset_init_early(void);
 extern int cpuset_init(void);
 extern void cpuset_init_smp(void);
-extern void cpuset_fork(struct task_struct *p);
-extern void cpuset_exit(struct task_struct *p);
 extern cpumask_t cpuset_cpus_allowed(struct task_struct *p);
 extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
 void cpuset_init_current_mems_allowed(void);
@@ -47,10 +46,6 @@ extern void __cpuset_memory_pressure_bum
 
 extern struct file_operations proc_cpuset_operations;
 extern char *cpuset_task_status_allowed(struct task_struct *task, char *buffer);
-
-extern void cpuset_lock(void);
-extern void cpuset_unlock(void);
-
 extern int cpuset_mem_spread_node(void);
 
 static inline int cpuset_do_page_mem_spread(void)
@@ -65,13 +60,22 @@ static inline int cpuset_do_slab_mem_spr
 
 extern void cpuset_track_online_nodes(void);
 
+extern int cpuset_can_attach_task(struct container *cont,
+				  struct task_struct *tsk);
+extern void cpuset_attach_task(struct container *cont,
+				struct task_struct *tsk);
+extern void cpuset_post_attach_task(struct container *cont,
+				    struct container *oldcont,
+				    struct task_struct *tsk);
+extern int cpuset_populate_dir(struct container *cont);
+extern int cpuset_create(struct container *cont);
+extern void cpuset_destroy(struct container *cont);
+
 #else /* !CONFIG_CPUSETS */
 
 static inline int cpuset_init_early(void) { return 0; }
 static inline int cpuset_init(void) { return 0; }
 static inline void cpuset_init_smp(void) {}
-static inline void cpuset_fork(struct task_struct *p) {}
-static inline void cpuset_exit(struct task_struct *p) {}
 
 static inline cpumask_t cpuset_cpus_allowed(struct task_struct *p)
 {
@@ -110,9 +114,6 @@ static inline char *cpuset_task_status_a
 	return buffer;
 }
 
-static inline void cpuset_lock(void) {}
-static inline void cpuset_unlock(void) {}
-
 static inline int cpuset_mem_spread_node(void)
 {
 	return 0;
kernel/exit.c +0 -2
Index: container-2.6.19-rc2/kernel/exit.c
===================================================================
@@ -29,7 +29,6 @@
 #include <linux/mempolicy.h>
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
-#include <linux/cpuset.h>
 #include <linux/container.h>
 #include <linux/syscalls.h>
 #include <linux/signal.h>
@@ -922,7 +921,6 @@ fastcall NORET_TYPE void do_exit(long co
 	__exit_files(tsk);
 	__exit_fs(tsk);
 	exit_thread();
-	cpuset_exit(tsk);
 	container_exit(tsk);
 	exit_keys(tsk);
 
kernel/fork.c +2 -5
Index: container-2.6.19-rc2/kernel/fork.c
===================================================================
@@ -30,7 +30,6 @@
 #include <linux/nsproxy.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
-#include <linux/cpuset.h>
 #include <linux/container.h>
 #include <linux/security.h>
 #include <linux/swap.h>
@@ -1055,13 +1054,12 @@ static struct task_struct *copy_process(
 	p->io_wait = NULL;
 	p->audit_context = NULL;
 	container_fork(p);
-	cpuset_fork(p);
 #ifdef CONFIG_NUMA
  	p->mempolicy = mpol_copy(p->mempolicy);
  	if (IS_ERR(p->mempolicy)) {
  		retval = PTR_ERR(p->mempolicy);
  		p->mempolicy = NULL;
- 		goto bad_fork_cleanup_cpuset;
+ 		goto bad_fork_cleanup_container;
  	}
 	mpol_fix_fork_child_flag(p);
 #endif
@@ -1287,9 +1285,8 @@ bad_fork_cleanup_security:
 bad_fork_cleanup_policy:
 #ifdef CONFIG_NUMA
 	mpol_free(p->mempolicy);
-bad_fork_cleanup_cpuset:
+bad_fork_cleanup_container:
 #endif
-	cpuset_exit(p);
 	container_exit(p);
 bad_fork_cleanup_delays_binfmt:
 	delayacct_tsk_free(p);
kernel/container.c +99 -6

213 lines not on this page · raw message

kernel/cpuset.c +147 -1174

1,781 lines not on this page · raw message

init/Kconfig +12 -7
Index: container-2.6.19-rc2/init/Kconfig
===================================================================
@@ -238,17 +238,12 @@ config IKCONFIG_PROC
 	  through /proc/config.gz.
 
 config CONTAINERS
-	bool "Container support"
-	help
-	  This option will let you create and manage process containers,
-	  which can be used to aggregate multiple processes, e.g. for
-	  the purposes of resource tracking.
-
-	  Say N if unsure
+	bool
 
 config CPUSETS
 	bool "Cpuset support"
 	depends on SMP
+	select CONTAINERS
 	help
 	  This option will let you create and manage CPUSETs which
 	  allow dynamically partitioning a system into sets of CPUs and
@@ -257,6 +252,16 @@ config CPUSETS
 
 	  Say N if unsure.
 
+config CPUSETS_LEGACY_API
+	bool "Maintain the existing cpuset API for generic container features"
+	depends on CPUSETS
+	default n
+	help
+	  This option preserves the following legacy CPUset behaviour:
+	  - /proc/<pid>/cpuset
+	  - defaults cpuset subsystem to enabled
+	  - defaults container release agent to /sbin/cpuset_release_agent
+
 config RELAY
 	bool "Kernel->user space relay support (formerly relayfs)"
 	help
mm/oom_kill.c +3 -3
Index: container-2.6.19-rc2/mm/oom_kill.c
===================================================================
@@ -394,7 +394,7 @@ void out_of_memory(struct zonelist *zone
 		show_mem();
 	}
 
-	cpuset_lock();
+	container_lock();
 	read_lock(&tasklist_lock);
 
 	/*
@@ -428,7 +428,7 @@ retry:
 		/* Found nothing?!?! Either we hang forever, or we panic. */
 		if (!p) {
 			read_unlock(&tasklist_lock);
-			cpuset_unlock();
+			container_unlock();
 			panic("Out of memory and no killable processes...\n");
 		}
 
@@ -440,7 +440,7 @@ retry:
 
 out:
 	read_unlock(&tasklist_lock);
-	cpuset_unlock();
+	container_unlock();
 
 	/*
 	 * Give "p" a good chance of killing itself before we
include/linux/sched.h +1 -3
Index: container-2.6.19-rc2/include/linux/sched.h
===================================================================
@@ -721,7 +721,6 @@ extern unsigned int max_cache_size;
 
 struct io_context;			/* See blkdev.h */
 struct container;
-struct cpuset;
 #define NGROUPS_SMALL		32
 #define NGROUPS_PER_BLOCK	((int)(PAGE_SIZE / sizeof(gid_t)))
 struct group_info {
@@ -1002,7 +1001,6 @@ struct task_struct {
 	short il_next;
 #endif
 #ifdef CONFIG_CPUSETS
-	struct cpuset *cpuset;
 	nodemask_t mems_allowed;
 	int cpuset_mems_generation;
 	int cpuset_mem_spread_rotor;
@@ -1433,7 +1431,7 @@ static inline int thread_group_empty(str
 /*
  * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
  * subscriptions and synchronises with wait4().  Also used in procfs.  Also
- * pins the final release of task.io_context.  Also protects ->cpuset.
+ * pins the final release of task.io_context.  Also protects ->container.
  *
  * Nests both inside and outside of read_lock(&tasklist_lock).
  * It must not be nested with write_lock_irq(&tasklist_lock),
Documentation/cpusets.txt +31 -50

166 lines not on this page · raw message

fs/super.c +0 -5
Index: container-2.6.19-rc2/fs/super.c
===================================================================
@@ -39,11 +39,6 @@
 #include <linux/mutex.h>
 #include <asm/uaccess.h>
 
-
-void get_filesystem(struct file_system_type *fs);
-void put_filesystem(struct file_system_type *fs);
-struct file_system_type *get_fs_type(const char *name);
-
 LIST_HEAD(super_blocks);
 DEFINE_SPINLOCK(sb_lock);
 
include/linux/fs.h +2 -0
Index: container-2.6.19-rc2/include/linux/fs.h
===================================================================
@@ -1864,6 +1864,8 @@ extern int vfs_fstat(unsigned int, struc
 
 extern int vfs_ioctl(struct file *, unsigned int, unsigned int, unsigned long);
 
+extern void get_filesystem(struct file_system_type *fs);
+extern void put_filesystem(struct file_system_type *fs);
 extern struct file_system_type *get_fs_type(const char *name);
 extern struct super_block *get_super(struct block_device *);
 extern struct super_block *user_get_super(dev_t);

--

#3 lore raw

Re: [ckrm-tech] [PATCH 2/6] Cpusets hooked into containers

Quoted 6 lines · [email protected] wrote:

This patch removes the process grouping code from the cpusets code, instead hooking it into the generic container system. This temporarily adds cpuset-specific code in kernel/container.c, which is removed by the next patch in the series.

I needed the following patches to get the cpuset code to compile. Inlining two patches makes it hard to distinguish between the patches and harder to read them, so I am attaching them along with this email.

Signature · 4 lines
	Regards,
	Balbir Singh,
	Linux Technology Center,
	IBM Software Labs

---

 kernel/cpuset.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
kernel/cpuset.c renamed from kernel/cpuset.c~fix-cpuset-guarantee-online-cpus-mems-in-subtree +3 -1
diff -puN kernel/cpuset.c~fix-cpuset-guarantee-online-cpus-mems-in-subtree kernel/cpuset.c
@@ -1280,10 +1280,12 @@ int __init cpuset_init(void)
 
 static void guarantee_online_cpus_mems_in_subtree(const struct cpuset *cur)
 {
+	struct container *cont;
 	struct cpuset *c;
 
 	/* Each of our child cpusets mems must be online */
-	list_for_each_entry(c, &cur->children, sibling) {
+	list_for_each_entry(cont, &cur->container->children, sibling) {
+		c = container_cs(cont);
 		guarantee_online_cpus_mems_in_subtree(c);
 		if (!cpus_empty(c->cpus_allowed))
 			guarantee_online_cpus(c, &c->cpus_allowed);

_

---

 fs/proc/base.c |    7 -------
 1 file changed, 7 deletions(-)
fs/proc/base.c renamed from fs/proc/base.c~fix-cpuset-proc-operations +0 -7
diff -puN fs/proc/base.c~fix-cpuset-proc-operations fs/proc/base.c
@@ -68,7 +68,6 @@
 #include <linux/security.h>
 #include <linux/ptrace.h>
 #include <linux/seccomp.h>
-#include <linux/cpuset.h>
 #include <linux/container.h>
 #include <linux/audit.h>
 #include <linux/poll.h>
@@ -1788,9 +1787,6 @@ static struct pid_entry tgid_base_stuff[
 #ifdef CONFIG_CONTAINERS
 	REG("container",  S_IRUGO, container),
 #endif
-#ifdef CONFIG_CPUSETS
-	REG("cpuset",     S_IRUGO, cpuset),
-#endif
 	INF("oom_score",  S_IRUGO, oom_score),
 	REG("oom_adj",    S_IRUGO|S_IWUSR, oom_adjust),
 #ifdef CONFIG_AUDITSYSCALL
@@ -2065,9 +2061,6 @@ static struct pid_entry tid_base_stuff[]
 #ifdef CONFIG_CONTAINERS
 	REG("container",  S_IRUGO, container),
 #endif
-#ifdef CONFIG_CPUSETS
-	REG("cpuset",    S_IRUGO, cpuset),
-#endif
 	INF("oom_score", S_IRUGO, oom_score),
 	REG("oom_adj",   S_IRUGO|S_IWUSR, oom_adjust),
 #ifdef CONFIG_AUDITSYSCALL

_

#4 lore raw

Re: [ckrm-tech] [PATCH 2/6] Cpusets hooked into containers

Quoted 4 lines · Balbir Singh

I needed the following patches to get the cpuset code to compile. Inlining two patches makes it hard to distinguish between the patches and harder to read them, so I am attaching them along with this email.

The first I missed due to not compiling with CONFIG_HOTPLUG_* - thanks for the patch.

For the second, the following change to fs/proc/base.c should have appeared in cpusets_using_containers.patch, but got left out due to quilt misusage. It basically makes "cpuset" an alias for "container" in the relevant /proc directories if CONFIG_CPUSETS_LEGACY_API is defined.

fs/proc/base.c +0 -0
@@ -69,7 +69,6 @@

#include <linux/ptrace.h>
#include <linux/seccomp.h>
#include <linux/container.h>
-#include <linux/cpuset.h>
#include <linux/audit.h>
#include <linux/poll.h>
#include <linux/nsproxy.h>
@@ -1784,8 +1783,8 @@ static struct pid_entry tgid_base_stuff[ #ifdef CONFIG_CONTAINERS
REG("container", S_IRUGO, container),
#endif
-#ifdef CONFIG_CPUSETS
- REG("cpuset", S_IRUGO, cpuset),
+#ifdef CONFIG_CPUSETS_LEGACY_API
+ REG("cpuset", S_IRUGO, container),

#endif
	INF("oom_score",  S_IRUGO, oom_score),
	REG("oom_adj",    S_IRUGO|S_IWUSR, oom_adjust),

@@ -2061,8 +2060,8 @@ static struct pid_entry tid_base_stuff[] #ifdef CONFIG_CONTAINERS
REG("container", S_IRUGO, container),
#endif
-#ifdef CONFIG_CPUSETS
- REG("cpuset", S_IRUGO, cpuset),
+#ifdef CONFIG_CPUSETS_LEGACY_API
+ REG("cpuset", S_IRUGO, container),

#endif
	INF("oom_score", S_IRUGO, oom_score),
	REG("oom_adj",   S_IRUGO|S_IWUSR, oom_adjust),

Paul

#5 lore raw

Re: [ckrm-tech] [PATCH 2/6] Cpusets hooked into containers

Quoted 3 lines · Paul M

It basically makes "cpuset" an alias for "container"
in the relevant /proc directories if CONFIG_CPUSETS_LEGACY_API is defined.

Paul M - I never replied to your initial CONFIG_CPUSETS_LEGACY_API patch proposal - sorry.

An aspect of this proposal never made sense to me, so I put it aside and went on to other things.

It is important to me that the current cpuset API be maintained. The cpuset API seems to be working well, for a number of users.

Occassionally I will agree to subtle API changes (see another thread concerning cpu_exclusive and sched_domain cpuset flags), but not anything likely to break user code outright, except under duress.

But I presume this CONFIG_CPUSETS_LEGACY_API option means I either get to build a kernel that supports the new container API, or a kernel that supports the old cpuset API. That does not seem useful to me.

We need to support both API's, at runtime, at the same time. Not a choice of API's at build time with a kernel CONFIG option.

Perhaps I am missing something ...

Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#6 lore raw

Re: [ckrm-tech] [PATCH 2/6] Cpusets hooked into containers

Quoted 4 lines · Paul Jackson

But I presume this CONFIG_CPUSETS_LEGACY_API option means I either get to build a kernel that supports the new container API, or a kernel that supports the old cpuset API. That does not seem useful to me.

No. The changes triggered by CONFIG_CPUSETS_LEGACY_API are:

- /proc/<pid>/cpuset is an alias for /proc/<pid>/containers
- a dummy "cpuset" filesystem exists
- mounting the "cpuset" filesystem causes:
- the container filesystem to be mounted in its place
- the release agent to be switched to /sbin/cpuset_release_agent
- defaults the "cpuset" container type to be enabled

Paul

#7 lore raw

Re: [ckrm-tech] [PATCH 2/6] Cpusets hooked into containers

In reply to Paul Menage

Paul Menage wrote:
[snip]

Quoted 6 lines

For the second, the following change to fs/proc/base.c should have appeared in cpusets_using_containers.patch, but got left out due to quilt misusage. It basically makes "cpuset" an alias for "container" in the relevant /proc directories if CONFIG_CPUSETS_LEGACY_API is defined.

[snip]

Yeah, this looks much better and more like the fix

Signature · 3 lines
	Balbir Singh,
	Linux Technology Center,
	IBM Software Labs
#8 lore raw

PATCH 3/6 Add generic multi-subsystem API to containers

In reply to [email protected]

This patch removes all cpuset-specific knowlege from the container system, replacing it with a generic API that can be used by multiple subsystems. Cpusets is adapted to be a container subsystem.

---
 Documentation/containers.txt |  160 ++++++++++++++++++++++++++++++++++++++-
 include/linux/container.h    |   43 +++++++++-
 include/linux/cpuset.h       |   11 --
 kernel/container.c           |  175 ++++++++++++++++++++++++++++++++++++-------
 kernel/cpuset.c              |  126 ++++++++++++++++++++----------
 5 files changed, 427 insertions(+), 88 deletions(-)
include/linux/container.h +40 -3
Index: container-2.6.19-rc2/include/linux/container.h
===================================================================
@@ -30,6 +30,8 @@ extern void container_unlock(void);
 extern void container_manage_lock(void);
 extern void container_manage_unlock(void);
 
+#define MAX_CONTAINER_SUBSYS 8
+
 struct container {
 	unsigned long flags;		/* "unsigned long" so bitops work */
 
@@ -48,9 +50,8 @@ struct container {
 	struct container *parent;	/* my parent */
 	struct dentry *dentry;		/* container fs entry */
 
-#ifdef CONFIG_CPUSETS
-	struct cpuset *cpuset;
-#endif
+	/* Private pointers for each registered subsystem */
+	void *subsys[MAX_CONTAINER_SUBSYS];
 };
 
 /* struct cftype:
@@ -87,6 +88,42 @@ int container_is_removed(const struct co
 void container_set_release_agent_path(const char *path);
 #endif
 
+/* Container subsystem type. See Documentation/containers.txt for details */
+
+struct container_subsys {
+	int (*create)(struct container_subsys *ss, struct container *cont);
+	void (*destroy)(struct container_subsys *ss, struct container *cont);
+	int (*can_attach)(struct container_subsys *ss,
+			  struct container *cont, struct task_struct *tsk);
+	void (*attach)(struct container_subsys *ss, struct container *cont,
+			struct container *old_cont, struct task_struct *tsk);
+	void (*post_attach)(struct container_subsys *ss,
+			    struct container *cont,
+			    struct container *old_cont,
+			    struct task_struct *tsk);
+	int (*populate)(struct container_subsys *ss,
+			struct container *cont);
+
+	int subsys_id;
+#define MAX_CONTAINER_TYPE_NAMELEN 32
+	const char *name;
+
+	/* enabled/disabled flag - managed by container.c, but can be
+	 * set to a default value before registration */
+	int enabled;
+
+	/* Fields below here are managed by container.c and should not
+	 * be accessed by subsystems. */
+
+	/* file handler for enable/disable */
+	struct cftype enable_cft;
+	char enable_cft_filename[MAX_CONTAINER_TYPE_NAMELEN + 10];
+
+
+};
+
+int container_register_subsys(struct container_subsys *subsys);
+
 #else /* !CONFIG_CONTAINERS */
 
 static inline int container_init_early(void) { return 0; }
include/linux/cpuset.h +0 -11
Index: container-2.6.19-rc2/include/linux/cpuset.h
===================================================================
@@ -60,17 +60,6 @@ static inline int cpuset_do_slab_mem_spr
 
 extern void cpuset_track_online_nodes(void);
 
-extern int cpuset_can_attach_task(struct container *cont,
-				  struct task_struct *tsk);
-extern void cpuset_attach_task(struct container *cont,
-				struct task_struct *tsk);
-extern void cpuset_post_attach_task(struct container *cont,
-				    struct container *oldcont,
-				    struct task_struct *tsk);
-extern int cpuset_populate_dir(struct container *cont);
-extern int cpuset_create(struct container *cont);
-extern void cpuset_destroy(struct container *cont);
-
 #else /* !CONFIG_CPUSETS */
 
 static inline int cpuset_init_early(void) { return 0; }
kernel/container.c +147 -28

277 lines not on this page · raw message

kernel/cpuset.c +85 -41

306 lines not on this page · raw message

Documentation/containers.txt +155 -5

182 lines not on this page · raw message

--

#9 lore raw

PATCH 4/6 Simple CPU accounting container subsystem

In reply to [email protected]

This demonstrates how to use the generic container subsystem for a simple resource tracker that counts the total CPU time used by all processes in a container, during the time that they're members of the container.

---
 include/linux/cpu_acct.h |   14 +++++
 init/Kconfig             |    7 ++
 kernel/Makefile          |    1 
 kernel/cpu_acct.c        |  110 +++++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched.c           |    6 ++
 5 files changed, 138 insertions(+)
include/linux/cpu_acct.h new file +14 -0
Index: container-2.6.19-rc2/include/linux/cpu_acct.h
===================================================================
@@ -0,0 +1,14 @@
+
+#ifndef _LINUX_CPU_ACCT_H
+#define _LINUX_CPU_ACCT_H
+
+#include <linux/container.h>
+#include <asm/cputime.h>
+
+#ifdef CONFIG_CONTAINER_CPUACCT
+extern void cpuacct_charge(struct task_struct *, cputime_t cputime);
+#else
+static void inline cpuacct_charge(struct task_struct *p, cputime_t cputime) {}
+#endif
+
+#endif
init/Kconfig +7 -0
Index: container-2.6.19-rc2/init/Kconfig
===================================================================
@@ -261,6 +261,13 @@ config CPUSETS_LEGACY_API
 	  - /proc/<pid>/cpuset
 	  - defaults cpuset subsystem to enabled
 
+config CONTAINER_CPUACCT
+	bool "Simple CPU accounting container subsystem"
+	select CONTAINERS
+	help
+	  Provides a simple Resource Controller for monitoring the
+	  total CPU consumed by the tasks in a container
+
 config RELAY
 	bool "Kernel->user space relay support (formerly relayfs)"
 	help
kernel/cpu_acct.c new file +110 -0

110 lines not on this page · raw message

kernel/Makefile +1 -0
Index: container-2.6.19-rc2/kernel/Makefile
===================================================================
@@ -38,6 +38,7 @@ obj-$(CONFIG_KEXEC) += kexec.o
 obj-$(CONFIG_COMPAT) += compat.o
 obj-$(CONFIG_CONTAINERS) += container.o
 obj-$(CONFIG_CPUSETS) += cpuset.o
+obj-$(CONFIG_CONTAINER_CPUACCT) += cpu_acct.o
 obj-$(CONFIG_IKCONFIG) += configs.o
 obj-$(CONFIG_STOP_MACHINE) += stop_machine.o
 obj-$(CONFIG_AUDIT) += audit.o auditfilter.o
kernel/sched.c +6 -0
Index: container-2.6.19-rc2/kernel/sched.c
===================================================================
@@ -52,6 +52,7 @@
 #include <linux/tsacct_kern.h>
 #include <linux/kprobes.h>
 #include <linux/delayacct.h>
+#include <linux/cpu_acct.h>
 #include <asm/tlb.h>
 
 #include <asm/unistd.h>
@@ -2977,6 +2978,8 @@ void account_user_time(struct task_struc
 
 	p->utime = cputime_add(p->utime, cputime);
 
+	cpuacct_charge(p, cputime);
+
 	/* Add user time to cpustat. */
 	tmp = cputime_to_cputime64(cputime);
 	if (TASK_NICE(p) > 0)
@@ -3000,6 +3003,9 @@ void account_system_time(struct task_str
 
 	p->stime = cputime_add(p->stime, cputime);
 
+	if (p != rq->idle)
+		cpuacct_charge(p, cputime);
+
 	/* Add system time to cpustat. */
 	tmp = cputime_to_cputime64(cputime);
 	if (hardirq_count() - hardirq_offset)

--

#10 lore raw

PATCH 5/6 Extension to container system to allow fork/exit callbacks

In reply to [email protected]

This patch adds fork/exit callbacks to container subsystems, and ensures that every registered container has received one fork callback for each task running int the system, and one exit callback for each task that exited since it was registered.

Since the fork/exit path is performance sensitive, an RCU-protected flag indicates to the fork/exit hooks whether they need to take the callback mutex and scan the list of registered subsystems to look for fork/exit handlers.

 Documentation/containers.txt |   13 +++++
 include/linux/container.h    |    4 +
 kernel/container.c           |  108 +++++++++++++++++++++++++++++++++++++++----
 3 files changed, 116 insertions(+), 9 deletions(-)
include/linux/container.h +4 -0
Index: container-2.6.19-rc2/include/linux/container.h
===================================================================
@@ -101,6 +101,10 @@ struct container_subsys {
 			    struct container *cont,
 			    struct container *old_cont,
 			    struct task_struct *tsk);
+	void (*fork)(struct container_subsys *ss, struct container *cont,
+		     struct task_struct *task);
+	void (*exit)(struct container_subsys *ss, struct container *cont,
+		     struct task_struct *task);
 	int (*populate)(struct container_subsys *ss,
 			struct container *cont);
 
kernel/container.c +99 -9

160 lines not on this page · raw message

Documentation/containers.txt +13 -0
Index: container-2.6.19-rc2/Documentation/containers.txt
===================================================================
@@ -353,6 +353,19 @@ LL=manage_mutex
 Called after the task has been attached to the container, to allow any
 post-attachment activity that requires memory allocations or blocking.
 
+void fork(struct container_subsy *ss, struct container *cont,
+	  struct task_struct *task)
+LL=callback_mutex, maybe read_lock(tasklist_lock)
+
+Called when a task is forked into a container. Also called during
+registration for all existing tasks.
+
+void exit(struct container_subsys *ss, struct container *cont,
+	  struct task_struct *task)
+LL=callback_mutex
+
+Called during task exit
+
 int populate(struct container_subsys *ss, struct container *cont)
 LL=none
 

--

#11 lore raw

PATCH 6/6 Resource Groups over generic containers

In reply to [email protected]

This patch provides the RG core and numtasks controller as container subsystems. The changes to the core involve primarily removing the group management, task membership and configfs support and adding interface layers to talk to the generic container layer instead.

Each resource controller becomes an independent container subsystem; the RG core is essentially a library that the resource controllers can use to provide the RG API to userspace. Rather than a single shares and stats file in each group, there's a <controller>_shares and a <controller>_stats file, each linked to the appropriate resource controller.

The example numtasks resource controller is almost unchanged; the primary differences involve locking, the details of which are still to be finalized.

 include/linux/moduleparam.h  |   12 -
 include/linux/numtasks.h     |   28 ++
 include/linux/res_group.h    |   82 +++++++
 include/linux/res_group_rc.h |   97 +++++++++
 init/Kconfig                 |   22 ++
 kernel/Makefile              |    1 
 kernel/fork.c                |    7 
 kernel/res_group/Makefile    |    2 
 kernel/res_group/local.h     |   38 +++
 kernel/res_group/numtasks.c  |  461 +++++++++++++++++++++++++++++++++++++++++++
 kernel/res_group/res_group.c |  161 +++++++++++++++
 kernel/res_group/rgcs.c      |  302 ++++++++++++++++++++++++++++
 kernel/res_group/shares.c    |  228 +++++++++++++++++++++
 13 files changed, 1437 insertions(+), 4 deletions(-)
include/linux/moduleparam.h +9 -3
Index: container-2.6.19-rc2/include/linux/moduleparam.h
===================================================================
@@ -75,11 +75,17 @@ struct kparam_array
 /* Helper functions: type is byte, short, ushort, int, uint, long,
    ulong, charp, bool or invbool, or XXX if you define param_get_XXX,
    param_set_XXX and param_check_XXX. */
-#define module_param_named(name, value, type, perm)			   \
-	param_check_##type(name, &(value));				   \
-	module_param_call(name, param_set_##type, param_get_##type, &value, perm); \
+#define module_param_named_call(name, value, type, set, perm)		\
+	param_check_##type(name, &(value));				\
+	module_param_call(name, set, param_get_##type, &(value), perm); \
 	__MODULE_PARM_TYPE(name, #type)
 
+#define module_param_named(name, value, type, perm)			   \
+	module_param_named_call(name, value, type, param_set_##type, perm)
+
+#define module_param_set_call(name, type, setfn, perm) \
+	module_param_named_call(name, name, type, setfn, perm)
+
 #define module_param(name, type, perm)				\
 	module_param_named(name, name, type, perm)
 
include/linux/numtasks.h new file +28 -0
Index: container-2.6.19-rc2/include/linux/numtasks.h
===================================================================
@@ -0,0 +1,28 @@
+/* numtasks.h - No. of tasks resource controller for Resource Groups
+ *
+ * Copyright (C) Chandra Seetharaman, IBM Corp. 2003, 2004, 2005
+ *
+ * Provides No. of tasks resource controller for Resource Groups
+ *
+ * Latest version, more details at http://ckrm.sf.net
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+#ifndef _LINUX_NUMTASKS_H
+#define _LINUX_NUMTASKS_H
+
+#ifdef CONFIG_RES_GROUPS_NUMTASKS
+#include <linux/res_group_rc.h>
+
+extern int numtasks_allow_fork(struct task_struct *);
+
+#else /* CONFIG_RES_GROUPS_NUMTASKS */
+
+#define numtasks_allow_fork(task) (0)
+
+#endif /* CONFIG_RES_GROUPS_NUMTASKS */
+#endif /* _LINUX_NUMTASKS_H */
include/linux/res_group.h new file +82 -0

82 lines not on this page · raw message

include/linux/res_group_rc.h new file +97 -0

97 lines not on this page · raw message

init/Kconfig +22 -0
Index: container-2.6.19-rc2/init/Kconfig
===================================================================
@@ -303,6 +303,28 @@ config TASK_XACCT
 
 	  Say N if unsure.
 
+menu "Resource Groups"
+
+config RES_GROUPS
+	bool "Resource Groups"
+	depends on EXPERIMENTAL
+	select CONTAINERS
+	help
+	  Resource Groups is a framework for controlling and monitoring
+	  resource allocation of user-defined groups of tasks. For more
+	  information, please visit http://ckrm.sf.net.
+
+config RES_GROUPS_NUMTASKS
+	bool "Number of Tasks Resource Controller"
+	depends on RES_GROUPS
+	default y
+	help
+	  Provides a Resource Controller for Resource Groups that allows
+	  limiting number of tasks a resource group can have.
+
+	  Say N if unsure, Y to use the feature.
+
+endmenu
 config SYSCTL
 	bool
 
kernel/Makefile +1 -0
Index: container-2.6.19-rc2/kernel/Makefile
===================================================================
@@ -53,6 +53,7 @@ obj-$(CONFIG_RELAY) += relay.o
 obj-$(CONFIG_UTS_NS) += utsname.o
 obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
+obj-$(CONFIG_RES_GROUPS) += res_group/
 
 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
 # According to Alan Modra <[email protected]>, the -fno-omit-frame-pointer is
kernel/fork.c +6 -1
Index: container-2.6.19-rc2/kernel/fork.c
===================================================================
@@ -48,6 +48,7 @@
 #include <linux/delayacct.h>
 #include <linux/taskstats_kern.h>
 #include <linux/random.h>
+#include <linux/numtasks.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -1353,7 +1354,7 @@ long do_fork(unsigned long clone_flags,
 	      int __user *child_tidptr)
 {
 	struct task_struct *p;
-	int trace = 0;
+	int trace = 0, rc;
 	struct pid *pid = alloc_pid();
 	long nr;
 
@@ -1366,6 +1367,10 @@ long do_fork(unsigned long clone_flags,
 			clone_flags |= CLONE_PTRACE;
 	}
 
+	rc = numtasks_allow_fork(current);
+	if (rc)
+		return rc;
+
 	p = copy_process(clone_flags, stack_start, regs, stack_size, parent_tidptr, child_tidptr, nr);
 	/*
 	 * Do this prior waking up the new thread - the thread pointer
kernel/res_group/Makefile new file +2 -0
Index: container-2.6.19-rc2/kernel/res_group/Makefile
===================================================================
@@ -0,0 +1,2 @@
+obj-y = res_group.o shares.o rgcs.o
+obj-$(CONFIG_RES_GROUPS_NUMTASKS) += numtasks.o
kernel/res_group/local.h new file +38 -0
Index: container-2.6.19-rc2/kernel/res_group/local.h
===================================================================
@@ -0,0 +1,38 @@
+/*
+ * Contains function definitions that are local to the Resource Groups.
+ * NOT to be included by controllers.
+ */
+
+#include <linux/res_group_rc.h>
+
+extern struct res_controller *get_controller_by_name(const char *);
+extern struct res_controller *get_controller_by_id(unsigned int);
+extern void put_controller(struct res_controller *);
+extern struct resource_group *alloc_res_group(struct resource_group *,
+							const char *);
+extern int free_res_group(struct resource_group *);
+extern void release_res_group(struct kref *);
+extern int set_controller_shares(struct resource_group *,
+			struct res_controller *, const struct res_shares *);
+/* Set shares for the given resource group and resource to default values */
+extern void set_shares_to_default(struct resource_group *,
+						struct res_controller *);
+extern void res_group_teardown(void);
+extern int set_res_group(pid_t, struct resource_group *);
+extern void move_tasks_to_parent(struct resource_group *);
+
+ssize_t res_group_file_read(struct container *cont,
+			    struct cftype *cft,
+			    struct file *file,
+			    char __user *buf,
+			    size_t nbytes, loff_t *ppos);
+ssize_t res_group_file_write(struct container *cont,
+			     struct cftype *cft,
+			     struct file *file,
+			     const char __user *userbuf,
+			     size_t nbytes, loff_t *ppos);
+
+enum {
+	RG_FILE_SHARES,
+	RG_FILE_STATS,
+};
kernel/res_group/numtasks.c new file +461 -0

461 lines not on this page · raw message

kernel/res_group/res_group.c new file +161 -0

161 lines not on this page · raw message

kernel/res_group/rgcs.c new file +302 -0

302 lines not on this page · raw message

kernel/res_group/shares.c new file +228 -0

228 lines not on this page · raw message

--