Skip to contentSkip to conversation

Generic container system

RFC PATCH In review [email protected] 2 participants 14 messages Sep 28, 2006 – Oct 2, 2006

Subsystems (8)
lore
Share & tools
lore
#0 lore raw

cover Generic container system

There have recently been various proposals floating around for resource management/accounting subsystems in the kernel, including Res Groups, User BeanCounters and others. These all need the basic abstraction of being able to group together multiple processes in an aggregate, in order to track/limit the resources permitted to those processes, and all implement this grouping in different ways.

Already existing in the kernel is the cpuset subsystem; this has a process grouping mechanism that is mature, tested, and well documented (particularly with regards to synchronization rules).

This patchset extracts the process grouping code from cpusets into a generic container system, and makes the cpusets code a client of the container system.

It also provides a very simple additional container subsystem to do per-container CPU usage accounting; this is primarily to demonstrate use of the container subsystem API, but is useful in its own right.

The change is implemented in four stages:

1) extract the process grouping code from cpusets into a standalone system

2) remove the process grouping code from cpusets and hook into the container system

3) convert the container system to present a generic API, and make cpusets a client of that API

4) add a simple CPU accounting container subsystem

The intention is that the various resource management efforts can also become container clients, with the result that:

- the userspace APIs are (somewhat) normalised

- it's easier to test out e.g. the ResGroups CPU controller in conjunction with the UBC memory controller

- the additional kernel footprint of any of the competing resource management systems is substantially reduced, since it doesn't need to provide process grouping/containment, hence improving their chances of getting into the kernel

Possible TODOs include:

- define a convention for populating the per-container directories so that different subsystems don't clash with one another

- provide higher-level primitives (e.g. an easy interface to seq_file) for files registered by subsystems.

- support subsystem deregistering

---

#1 lore raw

RFC PATCH 1/4 Generic container system abstracted from cpusets code

This patch creates a generic process container system based on (and parallel top) the cpusets code. At a coarse level it was created by copying kernel/cpuset.c, doing s/cpuset/container/g, and stripping out any code that was cpuset-specific rather than applicable to any process container subsystem.

---
 Documentation/containers.txt |  229 +++++++
 fs/proc/base.c               |   19 
 include/linux/container.h    |   96 +++
 include/linux/sched.h        |    5 
 include/linux/utsrelease.h   |    1 
 init/Kconfig                 |    9 
 init/main.c                  |    3 
 kernel/Makefile              |    1 
 kernel/container.c           | 1342 +++++++++++++++++++++++++++++++++++++++++++
 kernel/exit.c                |    2 
 kernel/fork.c                |    3 
 11 files changed, 1709 insertions(+), 1 deletion(-)
fs/proc/base.c +19 -0
Index: linux-2.6.18/fs/proc/base.c
===================================================================
@@ -69,6 +69,7 @@
 #include <linux/ptrace.h>
 #include <linux/seccomp.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/audit.h>
 #include <linux/poll.h>
 #include "internal.h"
@@ -124,6 +125,9 @@ enum pid_directory_inos {
 #ifdef CONFIG_CPUSETS
 	PROC_TGID_CPUSET,
 #endif
+#ifdef CONFIG_CONTAINERS
+	PROC_TGID_CONTAINER,
+#endif
 #ifdef CONFIG_SECURITY
 	PROC_TGID_ATTR,
 	PROC_TGID_ATTR_CURRENT,
@@ -167,6 +171,9 @@ enum pid_directory_inos {
 #ifdef CONFIG_CPUSETS
 	PROC_TID_CPUSET,
 #endif
+#ifdef CONFIG_CONTAINERS
+	PROC_TID_CONTAINER,
+#endif
 #ifdef CONFIG_SECURITY
 	PROC_TID_ATTR,
 	PROC_TID_ATTR_CURRENT,
@@ -235,6 +242,9 @@ static struct pid_entry tgid_base_stuff[
 #ifdef CONFIG_CPUSETS
 	E(PROC_TGID_CPUSET,    "cpuset",  S_IFREG|S_IRUGO),
 #endif
+#ifdef CONFIG_CONTAINERS
+	E(PROC_TGID_CONTAINER,    "container",  S_IFREG|S_IRUGO),
+#endif
 	E(PROC_TGID_OOM_SCORE, "oom_score",S_IFREG|S_IRUGO),
 	E(PROC_TGID_OOM_ADJUST,"oom_adj", S_IFREG|S_IRUGO|S_IWUSR),
 #ifdef CONFIG_AUDITSYSCALL
@@ -277,6 +287,9 @@ static struct pid_entry tid_base_stuff[]
 #ifdef CONFIG_CPUSETS
 	E(PROC_TID_CPUSET,     "cpuset",  S_IFREG|S_IRUGO),
 #endif
+#ifdef CONFIG_CONTAINERS
+	E(PROC_TID_CONTAINER,    "container",  S_IFREG|S_IRUGO),
+#endif
 	E(PROC_TID_OOM_SCORE,  "oom_score",S_IFREG|S_IRUGO),
 	E(PROC_TID_OOM_ADJUST, "oom_adj", S_IFREG|S_IRUGO|S_IWUSR),
 #ifdef CONFIG_AUDITSYSCALL
@@ -1819,6 +1832,12 @@ static struct dentry *proc_pident_lookup
 			inode->i_fop = &proc_cpuset_operations;
 			break;
 #endif
+#ifdef CONFIG_CONTAINERS
+		case PROC_TID_CONTAINER:
+		case PROC_TGID_CONTAINER:
+			inode->i_fop = &proc_container_operations;
+			break;
+#endif
 		case PROC_TID_OOM_SCORE:
 		case PROC_TGID_OOM_SCORE:
 			inode->i_fop = &proc_info_file_operations;
include/linux/container.h new file +96 -0

96 lines not on this page · raw message

include/linux/sched.h +4 -1
Index: linux-2.6.18/include/linux/sched.h
===================================================================
@@ -710,8 +710,8 @@ extern unsigned int max_cache_size;
 
 struct io_context;			/* See blkdev.h */
 void exit_io_context(void);
+struct container;
 struct cpuset;
-
 #define NGROUPS_SMALL		32
 #define NGROUPS_PER_BLOCK	((int)(PAGE_SIZE / sizeof(gid_t)))
 struct group_info {
@@ -979,6 +979,9 @@ struct task_struct {
 	int cpuset_mems_generation;
 	int cpuset_mem_spread_rotor;
 #endif
+#ifdef CONFIG_CONTAINERS
+	struct container *container;
+#endif
 	struct robust_list_head __user *robust_list;
 #ifdef CONFIG_COMPAT
 	struct compat_robust_list_head __user *compat_robust_list;
include/linux/utsrelease.h new file +1 -0
Index: linux-2.6.18/include/linux/utsrelease.h
===================================================================
@@ -0,0 +1 @@
+#define UTS_RELEASE "2.6.18"
init/Kconfig +9 -0
Index: linux-2.6.18/init/Kconfig
===================================================================
@@ -220,6 +220,15 @@ config IKCONFIG_PROC
 	  This option enables access to the kernel configuration file
 	  through /proc/config.gz.
 
+config CONTAINERS
+	bool "Container support"
+	help
+	  This option will let you create and manage process containers,
+	  which can be used to aggregate multiple processes, e.g. for
+	  the purposes of resource tracking.
+
+	  Say N if unsure
+
 config CPUSETS
 	bool "Cpuset support"
 	depends on SMP
init/main.c +3 -0
Index: linux-2.6.18/init/main.c
===================================================================
@@ -40,6 +40,7 @@
 #include <linux/writeback.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/efi.h>
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
@@ -545,6 +546,7 @@ asmlinkage void __init start_kernel(void
 	}
 #endif
 	vfs_caches_init_early();
+	container_init_early();
 	cpuset_init_early();
 	mem_init();
 	kmem_cache_init();
@@ -575,6 +577,7 @@ asmlinkage void __init start_kernel(void
 #ifdef CONFIG_PROC_FS
 	proc_root_init();
 #endif
+	container_init();
 	cpuset_init();
 	taskstats_init_early();
 	delayacct_init();
kernel/container.c new file +1342 -0

1,342 lines not on this page · raw message

kernel/exit.c +2 -0
Index: linux-2.6.18/kernel/exit.c
===================================================================
@@ -28,6 +28,7 @@
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/syscalls.h>
 #include <linux/signal.h>
 #include <linux/posix-timers.h>
@@ -919,6 +920,7 @@ fastcall NORET_TYPE void do_exit(long co
 	exit_namespace(tsk);
 	exit_thread();
 	cpuset_exit(tsk);
+	container_exit(tsk);
 	exit_keys(tsk);
 
 	if (group_dead && tsk->signal->leader)
kernel/fork.c +3 -0
Index: linux-2.6.18/kernel/fork.c
===================================================================
@@ -30,6 +30,7 @@
 #include <linux/capability.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
+#include <linux/container.h>
 #include <linux/security.h>
 #include <linux/swap.h>
 #include <linux/syscalls.h>
@@ -1044,6 +1045,7 @@ static struct task_struct *copy_process(
 	p->io_context = NULL;
 	p->io_wait = NULL;
 	p->audit_context = NULL;
+	container_fork(p);
 	cpuset_fork(p);
 #ifdef CONFIG_NUMA
  	p->mempolicy = mpol_copy(p->mempolicy);
@@ -1278,6 +1280,7 @@ bad_fork_cleanup_policy:
 bad_fork_cleanup_cpuset:
 #endif
 	cpuset_exit(p);
+	container_exit(p);
 bad_fork_cleanup_delays_binfmt:
 	delayacct_tsk_free(p);
 	if (p->binfmt)
kernel/Makefile +1 -0
Index: linux-2.6.18/kernel/Makefile
===================================================================
@@ -36,6 +36,7 @@ obj-$(CONFIG_PM) += power/
 obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o
 obj-$(CONFIG_KEXEC) += kexec.o
 obj-$(CONFIG_COMPAT) += compat.o
+obj-$(CONFIG_CONTAINERS) += container.o
 obj-$(CONFIG_CPUSETS) += cpuset.o
 obj-$(CONFIG_IKCONFIG) += configs.o
 obj-$(CONFIG_STOP_MACHINE) += stop_machine.o
Documentation/containers.txt new file +229 -0

229 lines not on this page · raw message

--

#2 lore raw

Re: [RFC][PATCH 1/4] Generic container system abstracted from cpusets code

Be sure to spell out, if you haven't already, that there is just a single container hierarchy, with each task attached to a single container in that hierarchy, and each subsystem (e.g. cpusets) adding its own special files to the common container hierarchy directories.

It is not the case that each subsystem using containers gets to concoct its own hierarchy, independent of any other subsystem.

init/Kconfig config IKCONFIG_PROC · line 228

Quoted 6 lines · Menage
init/Kconfig +6 -0
+config CONTAINERS
+	bool "Container support"
+	help
+	  This option will let you create and manage process containers,
+	  which can be used to aggregate multiple processes, e.g. for
+	  the purposes of resource tracking.

I wonder if this should be a configurable ... perhaps not.

If someone configures in one or more of the subsystems, such as cpusets, that requires CONTAINERS, then we need to enable containers. Otherwise we don't need to enable CONTAINERS. I don't see what user input we need here specific to CONTAINERS.

Quoted 2 lines

+ * Copyright (C) 2003 BULL SA.
+ * Copyright (C) 2004-2006 Silicon Graphics, Inc.

Perhaps you want to add a Google or Menage copyright here?

Quoted 2 lines

+ * 2003-10-22 Updates by Stephen Hemminger.
+ * 2004 May-July Rework by Paul Jackson.

Perhaps time for another log line here, such as:
+ * 2006 Generalized to containers by Paul Menage.

include/linux/container.h line 49

Quoted 5 lines
include/linux/container.h new file +5 -0
+	struct list_head sibling;	/* my parents children */
+	struct list_head children;	/* my children */
+
+	struct container *parent;		/* my parent */
+	struct dentry *dentry;		/* container fs entry */

Delete extra tab after semi-colon on parent line.

Quoted 1 line

+ 2) mount -t container none /dev/container

(Yes - this pertains to something this you took as is from the current cpuset docmentation ...) Please don't use 'none' for this argument to mount. Use some string relevant to the type of filesystem being mounted. This argument shows up in error messages from mount, and as it notes in the mount(8) man page, when speaking of the proc file system, though it applies here too:

The proc file system is not associated with a special device, and when mounting it, an arbitrary keyword, such as proc can be used instead of a device specification. (The customary choice none is less fortunate:
the error message `none busy' from umount can be confusing.)

For example, make the above:

mount -t container container /dev/container

Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#3 lore raw

Re: [RFC][PATCH 1/4] Generic container system abstracted from cpusets code

Quoted 9 lines · Paul Jackson

Be sure to spell out, if you haven't already, that there is just a single container hierarchy, with each task attached to a single container in that hierarchy, and each subsystem (e.g. cpusets) adding its own special files to the common container hierarchy directories.

It is not the case that each subsystem using containers gets to concoct its own hierarchy, independent of any other subsystem.

Well, it wouldn't be too hard to have the container code manage multiple container hierarchies in parallel if people wanted the ability to do this. You'd just replace the

struct container *container;

field in task_struct with

struct container *containers[MAX_CONTAINER_HIERARCHIES]

and have each instance of the container filesystem manage a different hierarchy. Perhaps to configure this from userspace you could do something like

mount -t container xxx -octype=cpuset /dev/cpuset
mount -t container yyy -octype=resgroup,ctype=cpuacct /mnt/rescontainer

i.e. mount one container hierarchy on /dev/cpuset with just the cpuset container type, and mount an independent hierarchy on /mnt/rescontainer with the resgroup and cpuacct container types attached to it. You'd probably want to enforce a rule that each container type could only be attached to a single hierarchy at a time.

Anyone have any thoughts on the usefulness/insanity of such an idea? The most useful part of it may be the ability to e.g. create a container hierarchy that explicitly doesn't cause e.g. multiple cpusets to be created, if you want to avoid the overhead that kicks in when there are multiple cpusets in use. That could be approximated without the overhead of multiple container hierarchy support by having a control file(s) in the top container dir that lets you configure which container subsystems are actually in use. So e.g. if you turned cpusets off before creating any subcontainers, all containers would actually share the same cpuset instance, and we could avoid any of the overheads associated with having multiple cpusets.

This would be somewhere in between the existing state of a single hierarchy with multiple fixed container types, and the full-blown multiple hierarchies with configurably-attachable container types that I outlined above.

Quoted 5 lines

If someone configures in one or more of the subsystems, such as cpusets, that requires CONTAINERS, then we need to enable containers. Otherwise we don't need to enable CONTAINERS. I don't see what user input we need here specific to CONTAINERS.

Fair enough - so have the various container types do "select CONTAINERS" rather than "depends on CONTAINERS"?

Quoted 11 lines
Quoted 3 lines

+ * Copyright (C) 2003 BULL SA.
+ * Copyright (C) 2004-2006 Silicon Graphics, Inc.

Perhaps you want to add a Google or Menage copyright here?

Quoted 2 lines

+ * 2003-10-22 Updates by Stephen Hemminger.
+ * 2004 May-July Rework by Paul Jackson.

Perhaps time for another log line here, such as:
+ * 2006 Generalized to containers by Paul Menage.

I already had something like that at the top - I've now made it clearer that these copyright statements came from the cpuset.c code from which this code was abstracted.

Paul

#4 lore raw

Re: [ckrm-tech] [RFC][PATCH 1/4] Generic container system abstracted from cpusets code

Quoted 1 line · Paul M

Anyone have any thoughts on the usefulness/insanity of such an idea?

[multiple hierarchies] I'd say let's pretend we don't know how to do them, until we have good reason otherwise. Having just a single resource hierarchy simplifies the thinking of these things. If doing so fits 90% of the needs, and can be stretched to get most of the remaining needs, then that simplification is well worth it. It might mean that a reasonable number of people can actually understand this stuff, which is always a nice property.

Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#5 lore raw

Re: [ckrm-tech] [RFC][PATCH 1/4] Generic container system abstracted from cpusets code

In reply to Paul Menage
Quoted 5 lines
Quoted 2 lines

Perhaps time for another log line here, such as:
+ * 2006 Generalized to containers by Paul Menage.

I already had something like that at the top - I've now made it clearer that ...

Sure - so long as you're happy with your Copyright statements, I'm happy.

Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#6 lore raw

RFC PATCH 2/4 Cpusets hooked into containers

In reply to [email protected]

This patch removes the process grouping code from the cpusets code, instead hooking it into the generic container system. This temporarily adds cpuset-specific code in kernel/container.c, which is removed by the next patch in the series.

---
 Documentation/cpusets.txt |   80 +-
 fs/proc/base.c            |   18 
 include/linux/container.h |    4 
 include/linux/cpuset.h    |   27 
 include/linux/sched.h     |    4 
 init/Kconfig              |    2 
 kernel/container.c        |   50 +
 kernel/cpuset.c           | 1317 ++++------------------------------------------
 kernel/exit.c             |    2 
 kernel/fork.c             |    7 
 mm/oom_kill.c             |    6 
 11 files changed, 228 insertions(+), 1289 deletions(-)
include/linux/container.h +4 -0
Index: linux-2.6.18/include/linux/container.h
===================================================================
@@ -47,6 +47,10 @@ struct container {
 
 	struct container *parent;		/* my parent */
 	struct dentry *dentry;		/* container fs entry */
+
+#ifdef CONFIG_CPUSETS
+	struct cpuset *cpuset;
+#endif
 };
 
 /* struct cftype:
include/linux/cpuset.h +13 -14
Index: linux-2.6.18/include/linux/cpuset.h
===================================================================
@@ -11,16 +11,13 @@
 #include <linux/sched.h>
 #include <linux/cpumask.h>
 #include <linux/nodemask.h>
+#include <linux/container.h>
 
 #ifdef CONFIG_CPUSETS
 
-extern int number_of_cpusets;	/* How many cpusets are defined in system? */
-
 extern int cpuset_init_early(void);
 extern int cpuset_init(void);
 extern void cpuset_init_smp(void);
-extern void cpuset_fork(struct task_struct *p);
-extern void cpuset_exit(struct task_struct *p);
 extern cpumask_t cpuset_cpus_allowed(struct task_struct *p);
 extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
 void cpuset_init_current_mems_allowed(void);
@@ -32,7 +29,7 @@ int cpuset_zonelist_valid_mems_allowed(s
 extern int __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask);
 static int inline cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
 {
-	return number_of_cpusets <= 1 || __cpuset_zone_allowed(z, gfp_mask);
+	return number_of_containers <= 1 || __cpuset_zone_allowed(z, gfp_mask);
 }
 
 extern int cpuset_excl_nodes_overlap(const struct task_struct *p);
@@ -47,10 +44,6 @@ extern void __cpuset_memory_pressure_bum
 
 extern struct file_operations proc_cpuset_operations;
 extern char *cpuset_task_status_allowed(struct task_struct *task, char *buffer);
-
-extern void cpuset_lock(void);
-extern void cpuset_unlock(void);
-
 extern int cpuset_mem_spread_node(void);
 
 static inline int cpuset_do_page_mem_spread(void)
@@ -63,13 +56,22 @@ static inline int cpuset_do_slab_mem_spr
 	return current->flags & PF_SPREAD_SLAB;
 }
 
+extern int cpuset_can_attach_task(struct container *cont,
+				  struct task_struct *tsk);
+extern void cpuset_attach_task(struct container *cont,
+				struct task_struct *tsk);
+extern void cpuset_post_attach_task(struct container *cont,
+				    struct container *oldcont,
+				    struct task_struct *tsk);
+extern int cpuset_populate_dir(struct container *cont);
+extern int cpuset_create(struct container *cont);
+extern void cpuset_destroy(struct container *cont);
+
 #else /* !CONFIG_CPUSETS */
 
 static inline int cpuset_init_early(void) { return 0; }
 static inline int cpuset_init(void) { return 0; }
 static inline void cpuset_init_smp(void) {}
-static inline void cpuset_fork(struct task_struct *p) {}
-static inline void cpuset_exit(struct task_struct *p) {}
 
 static inline cpumask_t cpuset_cpus_allowed(struct task_struct *p)
 {
@@ -108,9 +110,6 @@ static inline char *cpuset_task_status_a
 	return buffer;
 }
 
-static inline void cpuset_lock(void) {}
-static inline void cpuset_unlock(void) {}
-
 static inline int cpuset_mem_spread_node(void)
 {
 	return 0;
kernel/exit.c +0 -2
Index: linux-2.6.18/kernel/exit.c
===================================================================
@@ -27,7 +27,6 @@
 #include <linux/mempolicy.h>
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
-#include <linux/cpuset.h>
 #include <linux/container.h>
 #include <linux/syscalls.h>
 #include <linux/signal.h>
@@ -919,7 +918,6 @@ fastcall NORET_TYPE void do_exit(long co
 	__exit_fs(tsk);
 	exit_namespace(tsk);
 	exit_thread();
-	cpuset_exit(tsk);
 	container_exit(tsk);
 	exit_keys(tsk);
 
kernel/fork.c +2 -5
Index: linux-2.6.18/kernel/fork.c
===================================================================
@@ -29,7 +29,6 @@
 #include <linux/fs.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
-#include <linux/cpuset.h>
 #include <linux/container.h>
 #include <linux/security.h>
 #include <linux/swap.h>
@@ -1046,13 +1045,12 @@ static struct task_struct *copy_process(
 	p->io_wait = NULL;
 	p->audit_context = NULL;
 	container_fork(p);
-	cpuset_fork(p);
 #ifdef CONFIG_NUMA
  	p->mempolicy = mpol_copy(p->mempolicy);
  	if (IS_ERR(p->mempolicy)) {
  		retval = PTR_ERR(p->mempolicy);
  		p->mempolicy = NULL;
- 		goto bad_fork_cleanup_cpuset;
+ 		goto bad_fork_cleanup_container;
  	}
 	mpol_fix_fork_child_flag(p);
 #endif
@@ -1277,9 +1275,8 @@ bad_fork_cleanup_security:
 bad_fork_cleanup_policy:
 #ifdef CONFIG_NUMA
 	mpol_free(p->mempolicy);
-bad_fork_cleanup_cpuset:
+bad_fork_cleanup_container:
 #endif
-	cpuset_exit(p);
 	container_exit(p);
 bad_fork_cleanup_delays_binfmt:
 	delayacct_tsk_free(p);
kernel/container.c +47 -3

110 lines not on this page · raw message

kernel/cpuset.c +126 -1191

1,754 lines not on this page · raw message

init/Kconfig +1 -1
Index: linux-2.6.18/init/Kconfig
===================================================================
@@ -231,7 +231,7 @@ config CONTAINERS
 
 config CPUSETS
 	bool "Cpuset support"
-	depends on SMP
+	depends on SMP && CONTAINERS
 	help
 	  This option will let you create and manage CPUSETs which
 	  allow dynamically partitioning a system into sets of CPUs and
mm/oom_kill.c +3 -3
Index: linux-2.6.18/mm/oom_kill.c
===================================================================
@@ -326,7 +326,7 @@ void out_of_memory(struct zonelist *zone
 		show_mem();
 	}
 
-	cpuset_lock();
+	container_lock();
 	read_lock(&tasklist_lock);
 
 	/*
@@ -360,7 +360,7 @@ retry:
 		/* Found nothing?!?! Either we hang forever, or we panic. */
 		if (!p) {
 			read_unlock(&tasklist_lock);
-			cpuset_unlock();
+			container_unlock();
 			panic("Out of memory and no killable processes...\n");
 		}
 
@@ -372,7 +372,7 @@ retry:
 
 out:
 	read_unlock(&tasklist_lock);
-	cpuset_unlock();
+	container_unlock();
 
 	/*
 	 * Give "p" a good chance of killing itself before we
fs/proc/base.c +0 -18
Index: linux-2.6.18/fs/proc/base.c
===================================================================
@@ -122,9 +122,6 @@ enum pid_directory_inos {
 #ifdef CONFIG_SCHEDSTATS
 	PROC_TGID_SCHEDSTAT,
 #endif
-#ifdef CONFIG_CPUSETS
-	PROC_TGID_CPUSET,
-#endif
 #ifdef CONFIG_CONTAINERS
 	PROC_TGID_CONTAINER,
 #endif
@@ -168,9 +165,6 @@ enum pid_directory_inos {
 #ifdef CONFIG_SCHEDSTATS
 	PROC_TID_SCHEDSTAT,
 #endif
-#ifdef CONFIG_CPUSETS
-	PROC_TID_CPUSET,
-#endif
 #ifdef CONFIG_CONTAINERS
 	PROC_TID_CONTAINER,
 #endif
@@ -239,9 +233,6 @@ static struct pid_entry tgid_base_stuff[
 #ifdef CONFIG_SCHEDSTATS
 	E(PROC_TGID_SCHEDSTAT, "schedstat", S_IFREG|S_IRUGO),
 #endif
-#ifdef CONFIG_CPUSETS
-	E(PROC_TGID_CPUSET,    "cpuset",  S_IFREG|S_IRUGO),
-#endif
 #ifdef CONFIG_CONTAINERS
 	E(PROC_TGID_CONTAINER,    "container",  S_IFREG|S_IRUGO),
 #endif
@@ -284,9 +275,6 @@ static struct pid_entry tid_base_stuff[]
 #ifdef CONFIG_SCHEDSTATS
 	E(PROC_TID_SCHEDSTAT, "schedstat",S_IFREG|S_IRUGO),
 #endif
-#ifdef CONFIG_CPUSETS
-	E(PROC_TID_CPUSET,     "cpuset",  S_IFREG|S_IRUGO),
-#endif
 #ifdef CONFIG_CONTAINERS
 	E(PROC_TID_CONTAINER,    "container",  S_IFREG|S_IRUGO),
 #endif
@@ -1826,12 +1814,6 @@ static struct dentry *proc_pident_lookup
 			ei->op.proc_read = proc_pid_schedstat;
 			break;
 #endif
-#ifdef CONFIG_CPUSETS
-		case PROC_TID_CPUSET:
-		case PROC_TGID_CPUSET:
-			inode->i_fop = &proc_cpuset_operations;
-			break;
-#endif
 #ifdef CONFIG_CONTAINERS
 		case PROC_TID_CONTAINER:
 		case PROC_TGID_CONTAINER:
include/linux/sched.h +1 -3
Index: linux-2.6.18/include/linux/sched.h
===================================================================
@@ -711,7 +711,6 @@ extern unsigned int max_cache_size;
 struct io_context;			/* See blkdev.h */
 void exit_io_context(void);
 struct container;
-struct cpuset;
 #define NGROUPS_SMALL		32
 #define NGROUPS_PER_BLOCK	((int)(PAGE_SIZE / sizeof(gid_t)))
 struct group_info {
@@ -974,7 +973,6 @@ struct task_struct {
 	short il_next;
 #endif
 #ifdef CONFIG_CPUSETS
-	struct cpuset *cpuset;
 	nodemask_t mems_allowed;
 	int cpuset_mems_generation;
 	int cpuset_mem_spread_rotor;
@@ -1352,7 +1350,7 @@ static inline int thread_group_empty(str
 /*
  * Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
  * subscriptions and synchronises with wait4().  Also used in procfs.  Also
- * pins the final release of task.io_context.  Also protects ->cpuset.
+ * pins the final release of task.io_context.  Also protects ->container.
  *
  * Nests both inside and outside of read_lock(&tasklist_lock).
  * It must not be nested with write_lock_irq(&tasklist_lock),
Documentation/cpusets.txt +31 -50

166 lines not on this page · raw message

--

#7 lore raw

Re: [RFC][PATCH 2/4] Cpusets hooked into containers

kernel/container.c static long container_create(struct cont · line 1033

Quoted 4 lines · Menage
kernel/container.c +4 -0
+#ifdef CONFIG_CPUSETS
+	err = cpuset_create(cont);
+	if (err) goto err_unlock_free;
+#endif

I believe recommended kernel style puts the if() statement on a separate line, as in:

  +#ifdef CONFIG_CPUSETS
  +	err = cpuset_create(cont);
  +	if (err)
  +		goto err_unlock_free;
  +#endif
Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#8 lore raw

RFC PATCH 3/4 Add generic multi-subsystem API to containers

In reply to [email protected]

This patch removes all cpuset-specific knowlege from the container system, replacing it with a generic API that can be used by multiple subsystems. Cpusets is adapted to be a container subsystem.

---
 Documentation/containers.txt |  121 +++++++++++++++++++++++++++++++++++++++++--
 include/linux/container.h    |   24 +++++++-
 include/linux/cpuset.h       |   11 ---
 kernel/container.c           |   95 ++++++++++++++++++++++++---------
 kernel/cpuset.c              |  101 ++++++++++++++++++++++-------------
 5 files changed, 269 insertions(+), 83 deletions(-)
include/linux/container.h +21 -3
Index: linux-2.6.18/include/linux/container.h
===================================================================
@@ -30,6 +30,8 @@ extern void container_unlock(void);
 extern void container_manage_lock(void);
 extern void container_manage_unlock(void);
 
+#define MAX_CONTAINER_SUBSYS 8
+
 struct container {
 	unsigned long flags;		/* "unsigned long" so bitops work */
 
@@ -48,9 +50,8 @@ struct container {
 	struct container *parent;		/* my parent */
 	struct dentry *dentry;		/* container fs entry */
 
-#ifdef CONFIG_CPUSETS
-	struct cpuset *cpuset;
-#endif
+	/* Private pointers for each registered subsystem */
+	void *subsys[MAX_CONTAINER_SUBSYS];
 };
 
 /* struct cftype:
@@ -84,6 +85,23 @@ int container_add_file(struct container 
 
 int container_is_removed(const struct container *cont);
 
+/* Container subsystem type. See Documentation/containers.txt for details */
+
+struct container_subsys {
+	int (*create)(struct container *cont);
+	void (*destroy)(struct container *cont);
+	int (*can_attach)(struct container *cont, struct task_struct *tsk);
+	void (*attach)(struct container *cont, struct task_struct *tsk);
+	void (*post_attach)(struct container *cont,
+			   struct container *old_cont,
+			   struct task_struct *tsk);
+	int (*populate)(struct container *cont);
+
+	int subsys_id;
+};
+
+int container_register_subsys(struct container_subsys *subsys);
+
 #else /* !CONFIG_CONTAINERS */
 
 static inline int container_init_early(void) { return 0; }
include/linux/cpuset.h +0 -11
Index: linux-2.6.18/include/linux/cpuset.h
===================================================================
@@ -56,17 +56,6 @@ static inline int cpuset_do_slab_mem_spr
 	return current->flags & PF_SPREAD_SLAB;
 }
 
-extern int cpuset_can_attach_task(struct container *cont,
-				  struct task_struct *tsk);
-extern void cpuset_attach_task(struct container *cont,
-				struct task_struct *tsk);
-extern void cpuset_post_attach_task(struct container *cont,
-				    struct container *oldcont,
-				    struct task_struct *tsk);
-extern int cpuset_populate_dir(struct container *cont);
-extern int cpuset_create(struct container *cont);
-extern void cpuset_destroy(struct container *cont);
-
 #else /* !CONFIG_CPUSETS */
 
 static inline int cpuset_init_early(void) { return 0; }
kernel/container.c +68 -27

175 lines not on this page · raw message

kernel/cpuset.c +62 -39

260 lines not on this page · raw message

Documentation/containers.txt +118 -3

134 lines not on this page · raw message

--

#9 lore raw

RFC PATCH 4/4 Simple CPU accounting container subsystem

In reply to [email protected]

This demonstrates how to use the generic container subsystem for a simple resource tracker that counts the total CPU time used by all processes in a container, during the time that they're members of the container.

---
 include/linux/cpu_acct.h |   10 ++++
 init/Kconfig             |    8 +++
 kernel/Makefile          |    1 
 kernel/cpu_acct.c        |  104 +++++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched.c           |    6 ++
 5 files changed, 129 insertions(+)
include/linux/cpu_acct.h new file +10 -0
Index: linux-2.6.18/include/linux/cpu_acct.h
===================================================================
@@ -0,0 +1,10 @@
+
+#ifndef _LINUX_CPU_ACCT_H
+#define _LINUX_CPU_ACCT_H
+
+#include <linux/container.h>
+#include <asm/cputime.h>
+
+extern void cpuacct_charge(struct task_struct *, cputime_t cputime);
+
+#endif
init/Kconfig +8 -0
Index: linux-2.6.18/init/Kconfig
===================================================================
@@ -240,6 +240,14 @@ config CPUSETS
 
 	  Say N if unsure.
 
+config CONTAINER_CPUACCT
+	bool "Simple CPU accounting container subsystem"
+	depends on CONTAINERS
+	help
+	  Provides a simple Resource Controller for monitoring the
+	  total CPU consumed by the tasks in a container
+
+
 config RELAY
 	bool "Kernel->user space relay support (formerly relayfs)"
 	help
kernel/cpu_acct.c new file +104 -0

104 lines not on this page · raw message

kernel/Makefile +1 -0
Index: linux-2.6.18/kernel/Makefile
===================================================================
@@ -38,6 +38,7 @@ obj-$(CONFIG_KEXEC) += kexec.o
 obj-$(CONFIG_COMPAT) += compat.o
 obj-$(CONFIG_CONTAINERS) += container.o
 obj-$(CONFIG_CPUSETS) += cpuset.o
+obj-$(CONFIG_CONTAINER_CPUACCT) += cpu_acct.o
 obj-$(CONFIG_IKCONFIG) += configs.o
 obj-$(CONFIG_STOP_MACHINE) += stop_machine.o
 obj-$(CONFIG_AUDIT) += audit.o auditfilter.o
kernel/sched.c +6 -0
Index: linux-2.6.18/kernel/sched.c
===================================================================
@@ -52,6 +52,7 @@
 #include <linux/acct.h>
 #include <linux/kprobes.h>
 #include <linux/delayacct.h>
+#include <linux/cpu_acct.h>
 #include <asm/tlb.h>
 
 #include <asm/unistd.h>
@@ -2918,6 +2919,8 @@ void account_user_time(struct task_struc
 
 	p->utime = cputime_add(p->utime, cputime);
 
+	cpuacct_charge(p, cputime);
+
 	/* Add user time to cpustat. */
 	tmp = cputime_to_cputime64(cputime);
 	if (TASK_NICE(p) > 0)
@@ -2941,6 +2944,9 @@ void account_system_time(struct task_str
 
 	p->stime = cputime_add(p->stime, cputime);
 
+	if (p != rq->idle)
+		cpuacct_charge(p, cputime);
+
 	/* Add system time to cpustat. */
 	tmp = cputime_to_cputime64(cputime);
 	if (hardirq_count() - hardirq_offset)

--

#10 lore raw
In reply to [email protected]
Quoted 2 lines · Menage

This patchset extracts the process grouping code from cpusets into a generic container system,

Interesting. I'll try to give it a careful review in the next couple of days.

I've added Jay Lan <[email protected]> to the cc list. I encourage you to include him on your cc list in the future - thanks. He has expertise in some of SGI's earlier container-like efforts.

Quoted 2 lines

cpusets ... well documented
(particularly with regards to synchronization rules).

thanks ;

Question (perhaps already answered in your code - I haven't looked yet): can loadable kernel modules register containers? I'd like to see at least GPL modules be able to register containers, but I appreciate that this could be a controversial issue. Technically, I am guessing that some EXPORT_GPL_SYMPOL declarations on the necessary container registration routines would provide GPL modules with this capability.

Guess I should read the code ...

Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#11 lore raw
Quoted 3 lines · Paul Jackson

Question (perhaps already answered in your code - I haven't looked yet): can loadable kernel modules register containers? I'd like to

There's no reason in principle why not, although this patch doesn't export the symbols.

Paul

#12 lore raw
In reply to [email protected]

Aha - I must have missed the memo - I take it this patch set doesn't compile yet.

If I just apply it to 2.6.18, and try my favorite config 'make sn2_defconfig', then CONFIG_CPUSETS is disabled, because neither of the CONFIG_CONTAINER* settings are enabled, and furthermore it doesn't build, with the complaints:

===================================================================================
kernel/built-in.o: In function `account_system_time':
(.text+0xc1e2): undefined reference to cpuacct_charge' kernel/built-in.o: In function account_user_time':
(.text+0xc332): undefined reference to `cpuacct_charge'
make: *** [.tmp_vmlinux1] Error 1
===================================================================================

Apparently some generic code in kernel/sched.c is using cpuacct_charge, but cpuacct_charge is only defined if CONFIG_CONTAINER_CPUACCT is enabled. Perhaps you need an empty stub for cpuacct_charge in cpu_acct.h, in the event that CONTAINER_CPUACCT is not enabled.

If I then apply the following patch to my sn2_defconfig:

===================================================================================

arch/ia64/configs/sn2_defconfig 2006-09-28 21:05:22.119968786 -0700 renamed from arch/ia64/configs/sn2_defconfig 2006-09-28 20:37:02.162797053 -0700 +2 -0
@@ -1214,3 +1214,5 @@ CONFIG_CRYPTO_CRC32C=m
 #
 # Hardware crypto devices
 #
+CONFIG_CONTAINERS=y
+CONFIG_CONTAINER_CPUACCT=y

===================================================================================

it gets further (well, less far, actually), dying in the builds of kernel/cpu_acct.c and kernel/cpuset.o, with the complaints:

===================================================================================
kernel/cpu_acct.c: In function 'cpuusage_read':
kernel/cpu_acct.c:57: warning: format '%llu' expects type 'long long unsigned int', but argument 3 has type 'cputime64_t' kernel/cpu_acct.c:57: warning: format '%llu' expects type 'long long unsigned int', but argument 3 has type 'cputime64_t' kernel/cpuset.c:1000: warning: initialization from incompatible pointer type
kernel/cpuset.c:1007: warning: initialization from incompatible pointer type
kernel/cpuset.c:1014: warning: initialization from incompatible pointer type
kernel/cpuset.c:1021: warning: initialization from incompatible pointer type
kernel/cpuset.c:1028: warning: initialization from incompatible pointer type
kernel/cpuset.c:1035: warning: initialization from incompatible pointer type
kernel/cpuset.c:1042: warning: initialization from incompatible pointer type
kernel/cpuset.c:1049: warning: initialization from incompatible pointer type
kernel/cpuset.c:1056: warning: initialization from incompatible pointer type
kernel/cpuset.c: In function 'cpuset_create':
kernel/cpuset.c:1102: error: invalid lvalue in assignment
kernel/cpuset.c:1126: error: invalid lvalue in assignment
kernel/cpuset.c: At top level:
kernel/cpuset.c:1153: error: static declaration of 'cpuset_subsys' follows non-static declaration kernel/cpuset.c:58: error: previous declaration of 'cpuset_subsys' was here
make[1]: *** [kernel/cpuset.o] Error 1
===================================================================================

Looking at the relevant lines of kernel/cpuset.c, I see:

58: extern struct container_subsys cpuset_subsys;
59: #define container_cs(_cont) ((struct cpuset *)(_cont)->subsys[cpuset_subsys.subsys_id]) 1102: container_cs(cont) = &top_cpuset;
1126: container_cs(cont) = cs;
1153: static struct container_subsys cpuset_subsys = {

Looks like cpuset_subsys doesn't know if it is extern or static, and container_cs() is a cast thingie, which isn't allowed on the left side in modern gcc.

That's enough damage for one night ...

Signature · 3 lines
                  I won't rest till it's the best ...
                  Programmer, Linux Scalability
                  Paul Jackson <[email protected]> 1.925.600.0401
#13 lore raw
Quoted 5 lines · Paul Jackson

Apparently some generic code in kernel/sched.c is using cpuacct_charge, but cpuacct_charge is only defined if CONFIG_CONTAINER_CPUACCT is enabled. Perhaps you need an empty stub for cpuacct_charge in cpu_acct.h, in the event that CONTAINER_CPUACCT is not enabled.

You're right, the cpu_acct patch was a late example addition and I forgot to compile with CONFIG_CONTAINER_CPUACCT disabled ...

Quoted 4 lines

Looks like cpuset_subsys doesn't know if it is extern or static, and container_cs() is a cast thingie, which isn't allowed on the left side in modern gcc.

It builds fine on gcc 3.2.2, which is still a supported compiler according to Documentation/Changes - maybe that's a bit out of date?

Paul